21:03:59.683 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:03:59.683 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskRxIrUU/test-tmp
21:03:59.683 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
21:03:59.684 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskRxIrUU/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
21:03:59.685 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:04:00.016 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:04:00.016 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:04:00.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.019218 22308 mini_oidc.cc:150] Starting JWKS server
21:04:00.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.021646 22308 webserver.cc:492] Webserver started at http://127.0.0.1:42917/ using document root <none> and password file <none>
21:04:00.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.021981 22308 mini_oidc.cc:186] Starting OIDC Discovery server
21:04:00.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.022557 22308 webserver.cc:492] Webserver started at http://127.0.0.1:38059/ using document root <none> and password file <none>
21:04:00.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-28T21:04:00Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:04:00.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-28T21:04:00Z Disabled control of system clock
21:04:00.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.052685 22308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:00.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:00.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/wal
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/data
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/logs
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/data/info.pb
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.62:43803
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.62
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42917/jwks.json
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:04:00.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:04:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.380221 22319 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.380584 22319 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.380678 22319 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.388870 22319 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:04:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.388981 22319 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:04:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.389029 22319 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.389070 22319 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.389122 22319 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:04:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.389160 22319 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.389212 22319 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42917/jwks.json
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.399206 22319 master_runner.cc:386] Master server non-default flags:
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/data
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/wal
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.62:43803
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42917/jwks.json
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/data/info.pb
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.62
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/logs
21:04:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:00.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.400914 22319 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:00.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.402740 22319 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:00.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.411425 22324 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:00.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.411545 22325 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:00.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.411798 22327 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:00.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.413024 22319 server_base.cc:1047] running on GCE node
21:04:00.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.413820 22319 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:00.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.415292 22319 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:00.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.416545 22319 hybrid_clock.cc:648] HybridClock initialized: now 1761685440416475 us; error 47 us; skew 500 ppm
21:04:00.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.416908 22319 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:00.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.419255 22319 webserver.cc:492] Webserver started at http://127.21.201.62:35443/ using document root <none> and password file <none>
21:04:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.420029 22319 fs_manager.cc:362] Metadata directory not provided
21:04:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.420171 22319 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.420619 22319 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:00.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.423115 22319 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/data/instance:
21:04:00.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ae294c30f584b8290a509ce13a3d403"
21:04:00.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:00 on dist-test-slave-vx1b"
21:04:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.423930 22319 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/wal/instance:
21:04:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ae294c30f584b8290a509ce13a3d403"
21:04:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:00 on dist-test-slave-vx1b"
21:04:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.429733 22319 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.001s
21:04:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.433321 22333 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:00.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.434695 22319 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.434887 22319 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/wal
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ae294c30f584b8290a509ce13a3d403"
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:00 on dist-test-slave-vx1b"
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.435123 22319 fs_report.cc:389] FS layout report
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/wal
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/wal
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/data/data
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:00.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.475204 22319 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.476236 22319 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.476598 22319 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.500329 22319 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.62:43803
21:04:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.500389 22394 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.62:43803 every 8 connection(s)
21:04:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.501931 22319 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/data/info.pb
21:04:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.506158 22395 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.510725 22308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 22319
21:04:00.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.511078 22308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-0/wal/instance
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.514132 22308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/wal
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/data
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/logs
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/data/info.pb
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.61:41997
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.61
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42917/jwks.json
21:04:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:04:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:04:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.513527 22395 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } has no permanent_uuid. Determining permanent_uuid...
21:04:00.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.531069 22395 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } has no permanent_uuid. Determining permanent_uuid...
21:04:00.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.532189 22337 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.201.61:41997: connect: Connection refused (error 111)
21:04:00.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.533699 22395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.61:41997: Network error: Client connection negotiation failed: client connection to 127.21.201.61:41997: connect: Connection refused (error 111)
21:04:00.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.583045 22395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } attempt: 1
21:04:00.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.585109 22395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.61:41997: Network error: Client connection negotiation failed: client connection to 127.21.201.61:41997: connect: Connection refused (error 111)
21:04:00.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.653383 22395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } attempt: 2
21:04:00.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.655119 22395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.61:41997: Network error: Client connection negotiation failed: client connection to 127.21.201.61:41997: connect: Connection refused (error 111)
21:04:00.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.746495 22395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } attempt: 3
21:04:00.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.748405 22395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.61:41997: Network error: Client connection negotiation failed: client connection to 127.21.201.61:41997: connect: Connection refused (error 111)
21:04:00.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.848968 22398 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:00.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.849339 22398 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:00.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.849467 22398 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:00.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.857750 22398 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:04:00.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.857869 22398 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:04:00.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.857950 22398 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:00.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.858002 22398 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:00.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.858084 22398 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:04:00.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.858148 22398 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:00.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.858222 22398 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42917/jwks.json
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.868206 22398 master_runner.cc:386] Master server non-default flags:
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/data
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/wal
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.61:41997
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42917/jwks.json
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/data/info.pb
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.61
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/logs
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:00.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.870260 22398 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.872038 22398 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.879978 22405 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.879938 22406 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.880337 22408 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:00.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.881492 22398 server_base.cc:1047] running on GCE node
21:04:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.882264 22398 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.883695 22398 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:00.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.884900 22398 hybrid_clock.cc:648] HybridClock initialized: now 1761685440884817 us; error 60 us; skew 500 ppm
21:04:00.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.885206 22398 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:00.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.887303 22398 webserver.cc:492] Webserver started at http://127.21.201.61:43775/ using document root <none> and password file <none>
21:04:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.887955 22398 fs_manager.cc:362] Metadata directory not provided
21:04:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.888055 22398 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.888358 22398 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.890646 22398 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/data/instance:
21:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4eded57c27746c0a0c071bf5e3657c2"
21:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:00 on dist-test-slave-vx1b"
21:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.891351 22398 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/wal/instance:
21:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4eded57c27746c0a0c071bf5e3657c2"
21:04:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:00 on dist-test-slave-vx1b"
21:04:00.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.891788 22395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } attempt: 4
21:04:00.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.893705 22395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.61:41997: Network error: Client connection negotiation failed: client connection to 127.21.201.61:41997: connect: Connection refused (error 111)
21:04:00.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.896998 22398 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
21:04:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.900229 22414 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:00.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.901539 22398 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
21:04:00.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.901672 22398 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/wal
21:04:00.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4eded57c27746c0a0c071bf5e3657c2"
21:04:00.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:00 on dist-test-slave-vx1b"
21:04:00.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.901836 22398 fs_report.cc:389] FS layout report
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/wal
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/wal
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/data/data
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:00.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.915555 22398 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.916477 22398 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.916772 22398 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:00.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.940825 22398 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.61:41997
21:04:00.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.940847 22475 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.61:41997 every 8 connection(s)
21:04:00.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.942384 22398 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/data/info.pb
21:04:00.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.946780 22476 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.952585 22308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 22398
21:04:00.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.952791 22308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-1/wal/instance
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.953238 22476 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } has no permanent_uuid. Determining permanent_uuid...
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.954062 22308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/wal
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/data
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/logs
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/data/info.pb
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.60:33991
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.60
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42917/jwks.json
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
21:04:00.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
21:04:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.968217 22476 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } has no permanent_uuid. Determining permanent_uuid...
21:04:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:00.978386 22476 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } has no permanent_uuid. Determining permanent_uuid...
21:04:00.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.979614 22417 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:00.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:00.981333 22476 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.60:33991: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.030721 22476 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } attempt: 1
21:04:01.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.032886 22476 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.60:33991: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.101126 22476 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } attempt: 2
21:04:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.102916 22476 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.60:33991: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:01.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.192994 22395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } attempt: 5
21:04:01.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.194167 22476 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } attempt: 3
21:04:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.195902 22476 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.60:33991: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:01.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.199569 22395 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } has no permanent_uuid. Determining permanent_uuid...
21:04:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.201472 22395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.60:33991: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:01.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.252681 22395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } attempt: 1
21:04:01.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.254786 22395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.60:33991: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:01.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.295322 22479 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:01.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.295660 22479 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:01.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.295737 22479 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.303885 22479 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:04:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.303988 22479 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
21:04:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.304036 22479 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.304080 22479 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:04:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.304132 22479 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:04:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.304178 22479 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:04:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.304221 22479 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42917/jwks.json
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.314239 22479 master_runner.cc:386] Master server non-default flags:
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/data
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/wal
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.60:33991
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42917/jwks.json
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/data/info.pb
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.60
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/logs
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.315882 22479 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:01.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.317687 22479 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.323033 22395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } attempt: 2
21:04:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.325314 22395 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.60:33991: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.327602 22487 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.327638 22489 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.327682 22486 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.328683 22479 server_base.cc:1047] running on GCE node
21:04:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.329494 22479 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:01.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.330942 22479 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:01.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.332166 22479 hybrid_clock.cc:648] HybridClock initialized: now 1761685441332089 us; error 52 us; skew 500 ppm
21:04:01.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.332571 22479 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:01.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.334906 22479 webserver.cc:492] Webserver started at http://127.21.201.60:44099/ using document root <none> and password file <none>
21:04:01.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.335664 22479 fs_manager.cc:362] Metadata directory not provided
21:04:01.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.335810 22479 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.336174 22479 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.338647 22479 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/data/instance:
21:04:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37749401a30c4993b227bfe5d89d3478"
21:04:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:01 on dist-test-slave-vx1b"
21:04:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.339094 22476 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } attempt: 4
21:04:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.339494 22479 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/wal/instance:
21:04:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37749401a30c4993b227bfe5d89d3478"
21:04:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:01 on dist-test-slave-vx1b"
21:04:01.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.340874 22476 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.60:33991: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.345106 22479 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.008s sys 0.000s
21:04:01.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.348546 22495 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.349862 22479 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.350054 22479 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/wal
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37749401a30c4993b227bfe5d89d3478"
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:01 on dist-test-slave-vx1b"
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.350272 22479 fs_report.cc:389] FS layout report
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/wal
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/wal
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/data/data
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.366947 22479 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.367858 22479 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.368187 22479 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.391120 22479 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.60:33991
21:04:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.391151 22556 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.60:33991 every 8 connection(s)
21:04:01.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.392812 22479 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/data/info.pb
21:04:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.397307 22557 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.403071 22308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 22479
21:04:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.403268 22308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/master-2/wal/instance
21:04:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.403337 22557 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } has no permanent_uuid. Determining permanent_uuid...
21:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.417814 22557 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } has no permanent_uuid. Determining permanent_uuid...
21:04:01.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.424924 22557 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } has no permanent_uuid. Determining permanent_uuid...
21:04:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.432529 22395 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } attempt: 3
21:04:01.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.437948 22557 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478: Bootstrap starting.
21:04:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.441737 22557 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478: Neither blocks nor log segments found. Creating new log.
21:04:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.442772 22395 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403: Bootstrap starting.
21:04:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.442793 22557 log.cc:826] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478: Log is configured to *not* fsync() on all Append() calls
21:04:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.445873 22557 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478: No bootstrap required, opened a new log
21:04:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.446230 22395 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403: Neither blocks nor log segments found. Creating new log.
21:04:01.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.447192 22395 log.cc:826] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403: Log is configured to *not* fsync() on all Append() calls
21:04:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.449558 22395 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403: No bootstrap required, opened a new log
21:04:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.451449 22557 raft_consensus.cc:359] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:01.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.451962 22557 raft_consensus.cc:385] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:01.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.452172 22557 raft_consensus.cc:740] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 37749401a30c4993b227bfe5d89d3478, State: Initialized, Role: FOLLOWER
21:04:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.452775 22557 consensus_queue.cc:260] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.453658 22567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.453955 22567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [sys.catalog]: This master's current role is: FOLLOWER
21:04:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.454537 22557 sys_catalog.cc:565] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [sys.catalog]: configured and running, proceeding with master startup.
21:04:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.455682 22395 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.456099 22395 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.456245 22395 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5ae294c30f584b8290a509ce13a3d403, State: Initialized, Role: FOLLOWER
21:04:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.457172 22395 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.461032 22573 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.461462 22573 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: This master's current role is: FOLLOWER
21:04:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.462591 22395 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: configured and running, proceeding with master startup.
21:04:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.465376 22579 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:04:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.465603 22579 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:04:01.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.462847 22479 master_runner.cc:186] Error getting master registration for 127.21.201.62:43803: OK, instance_id { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" instance_seqno: 1761685440494823 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:04:01.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.469527 22479 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.477734 22591 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:04:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.478032 22591 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:04:01.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:01.479287 22339 tablet.cc:2378] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:01.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.474676 22319 master_runner.cc:186] Error getting master registration for 127.21.201.61:41997: OK, instance_id { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" instance_seqno: 1761685440935191 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:04:01.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.481164 22319 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.515302 22567 raft_consensus.cc:493] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.515646 22567 raft_consensus.cc:515] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.517884 22567 leader_election.cc:290] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5ae294c30f584b8290a509ce13a3d403 (127.21.201.62:43803), d4eded57c27746c0a0c071bf5e3657c2 (127.21.201.61:41997)
21:04:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.519105 22360 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "37749401a30c4993b227bfe5d89d3478" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5ae294c30f584b8290a509ce13a3d403" is_pre_election: true
21:04:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.519658 22360 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 37749401a30c4993b227bfe5d89d3478 in term 0.
21:04:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.520368 22499 leader_election.cc:304] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 37749401a30c4993b227bfe5d89d3478, 5ae294c30f584b8290a509ce13a3d403; no voters:
21:04:01.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.521037 22567 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:01.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.521202 22567 raft_consensus.cc:493] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:01.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.521306 22567 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 0 FOLLOWER]: Advancing to term 1
21:04:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.524693 22567 raft_consensus.cc:515] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:01.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.527173 22360 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "37749401a30c4993b227bfe5d89d3478" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5ae294c30f584b8290a509ce13a3d403"
21:04:01.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.527391 22360 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 0 FOLLOWER]: Advancing to term 1
21:04:01.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.527498 22567 leader_election.cc:290] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [CANDIDATE]: Term 1 election: Requested vote from peers 5ae294c30f584b8290a509ce13a3d403 (127.21.201.62:43803), d4eded57c27746c0a0c071bf5e3657c2 (127.21.201.61:41997)
21:04:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.531020 22360 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 37749401a30c4993b227bfe5d89d3478 in term 1.
21:04:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.531467 22499 leader_election.cc:304] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 37749401a30c4993b227bfe5d89d3478, 5ae294c30f584b8290a509ce13a3d403; no voters:
21:04:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.532055 22567 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 1 FOLLOWER]: Leader election won for term 1
21:04:01.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.541625 22567 raft_consensus.cc:697] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [term 1 LEADER]: Becoming Leader. State: Replica: 37749401a30c4993b227bfe5d89d3478, State: Running, Role: LEADER
21:04:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.542881 22567 consensus_queue.cc:237] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.549923 22593 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 37749401a30c4993b227bfe5d89d3478. Latest consensus state: current_term: 1 leader_uuid: "37749401a30c4993b227bfe5d89d3478" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.551049 22593 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [sys.catalog]: This master's current role is: LEADER
21:04:01.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.560216 22595 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:04:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.562948 22595 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:04:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.572217 22360 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 1 FOLLOWER]: Refusing update from remote peer 37749401a30c4993b227bfe5d89d3478: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.573488 22593 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:01.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.596938 22596 mvcc.cc:204] Tried to move back new op lower bound from 7215863568669065216 to 7215863568570089472. Current Snapshot: MvccSnapshot[applied={T|T < 7215863568669065216}]
21:04:01.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.596853 22573 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 37749401a30c4993b227bfe5d89d3478. Latest consensus state: current_term: 1 leader_uuid: "37749401a30c4993b227bfe5d89d3478" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:01.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.597126 22573 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: This master's current role is: FOLLOWER
21:04:01.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.597757 22567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "37749401a30c4993b227bfe5d89d3478" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:01.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.598006 22567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [sys.catalog]: This master's current role is: LEADER
21:04:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.602066 22595 catalog_manager.cc:1357] Generated new cluster ID: c74d27e67cf645a686abcaaa16a25183
21:04:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.602175 22595 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:04:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.613123 22573 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "37749401a30c4993b227bfe5d89d3478" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.613384 22573 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: This master's current role is: FOLLOWER
21:04:01.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.640053 22476 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } attempt: 5
21:04:01.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.654515 22476 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2: Bootstrap starting.
21:04:01.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.668167 22476 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2: Neither blocks nor log segments found. Creating new log.
21:04:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.671535 22476 log.cc:826] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2: Log is configured to *not* fsync() on all Append() calls
21:04:01.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.692971 22595 catalog_manager.cc:1380] Generated new certificate authority record
21:04:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.695446 22595 catalog_manager.cc:1514] Loading token signing keys...
21:04:01.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.696341 22476 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2: No bootstrap required, opened a new log
21:04:01.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.717224 22476 raft_consensus.cc:359] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:01.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.718052 22476 raft_consensus.cc:385] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:01.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.718334 22476 raft_consensus.cc:740] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d4eded57c27746c0a0c071bf5e3657c2, State: Initialized, Role: FOLLOWER
21:04:01.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.720073 22476 consensus_queue.cc:260] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.740552 22595 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478: Generated new TSK 0
21:04:01.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.742314 22595 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:04:01.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.745246 22603 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:01.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.745939 22603 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: This master's current role is: FOLLOWER
21:04:01.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.746346 22603 raft_consensus.cc:493] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:01.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.746747 22603 raft_consensus.cc:515] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.752218 22308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/wal
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/data
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/logs
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/data/info.pb
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.1:0
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.1
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.1
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:04:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.753986 22439 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 0 FOLLOWER]: Advancing to term 1
21:04:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.756528 22440 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "37749401a30c4993b227bfe5d89d3478" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d4eded57c27746c0a0c071bf5e3657c2"
21:04:01.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.757351 22441 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "37749401a30c4993b227bfe5d89d3478" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d4eded57c27746c0a0c071bf5e3657c2" is_pre_election: true
21:04:01.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.759068 22439 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 1 FOLLOWER]: Refusing update from remote peer 37749401a30c4993b227bfe5d89d3478: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.759732 22476 sys_catalog.cc:565] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: configured and running, proceeding with master startup.
21:04:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.759912 22567 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.761963 22360 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d4eded57c27746c0a0c071bf5e3657c2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5ae294c30f584b8290a509ce13a3d403" is_pre_election: true
21:04:01.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.763412 22522 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d4eded57c27746c0a0c071bf5e3657c2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "37749401a30c4993b227bfe5d89d3478" is_pre_election: true
21:04:01.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.764555 22417 leader_election.cc:304] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d4eded57c27746c0a0c071bf5e3657c2; no voters: 37749401a30c4993b227bfe5d89d3478, 5ae294c30f584b8290a509ce13a3d403
21:04:01.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.762809 22603 leader_election.cc:290] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5ae294c30f584b8290a509ce13a3d403 (127.21.201.62:43803), 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991)
21:04:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.794502 22606 mvcc.cc:204] Tried to move back new op lower bound from 7215863568669065216 to 7215863568570089472. Current Snapshot: MvccSnapshot[applied={T|T < 7215863568669065216}]
21:04:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.807561 22603 raft_consensus.cc:2749] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:04:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.808562 22605 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "37749401a30c4993b227bfe5d89d3478" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.808884 22605 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: This master's current role is: FOLLOWER
21:04:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.808744 22602 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 37749401a30c4993b227bfe5d89d3478. Latest consensus state: current_term: 1 leader_uuid: "37749401a30c4993b227bfe5d89d3478" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.809103 22602 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: This master's current role is: FOLLOWER
21:04:01.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.834484 22620 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:04:01.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.857774 22567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "37749401a30c4993b227bfe5d89d3478" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:01.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:01.858145 22567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37749401a30c4993b227bfe5d89d3478 [sys.catalog]: This master's current role is: LEADER
21:04:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.159263 22607 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.159591 22607 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.159734 22607 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.168190 22607 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.168375 22607 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.168464 22607 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.168536 22607 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.168664 22607 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.201.1
21:04:02.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.178545 22607 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:02.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:02.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:02.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:02.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:02.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/data
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/wal
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.1:0
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/data/info.pb
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.1
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.1
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/logs
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:02.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.180461 22607 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:02.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.182631 22607 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:02.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.191638 22626 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:02.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.191634 22625 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:02.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.192380 22628 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:02.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.192929 22607 server_base.cc:1047] running on GCE node
21:04:02.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.193477 22607 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:02.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.194322 22607 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:02.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.195533 22607 hybrid_clock.cc:648] HybridClock initialized: now 1761685442195477 us; error 39 us; skew 500 ppm
21:04:02.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.195870 22607 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:02.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.198117 22607 webserver.cc:492] Webserver started at http://127.21.201.1:42113/ using document root <none> and password file <none>
21:04:02.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.198844 22607 fs_manager.cc:362] Metadata directory not provided
21:04:02.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.198971 22607 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:02.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.199329 22607 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:02.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.201668 22607 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/data/instance:
21:04:02.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "832e52648dfc4f899b53b9ca1cb02855"
21:04:02.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:02 on dist-test-slave-vx1b"
21:04:02.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.202450 22607 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/wal/instance:
21:04:02.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "832e52648dfc4f899b53b9ca1cb02855"
21:04:02.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:02 on dist-test-slave-vx1b"
21:04:02.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.207984 22607 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.001s
21:04:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.211360 22634 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.212771 22607 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.212939 22607 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/wal
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "832e52648dfc4f899b53b9ca1cb02855"
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:02 on dist-test-slave-vx1b"
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.213135 22607 fs_report.cc:389] FS layout report
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/wal
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/wal
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/data/data
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:02.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.242291 22607 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:02.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.243165 22607 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:02.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.243464 22607 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:02.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.246526 22607 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:02.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.246637 22607 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:04:02.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.246753 22607 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:02.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.246934 22607 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.001s
21:04:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.317021 22607 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.1:33341
21:04:02.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.317075 22758 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.1:33341 every 8 connection(s)
21:04:02.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.318382 22607 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/data/info.pb
21:04:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.322487 22759 heartbeater.cc:344] Connected to a master server at 127.21.201.62:43803
21:04:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.323083 22759 heartbeater.cc:461] Registering TS with master...
21:04:02.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.324581 22759 heartbeater.cc:507] Master 127.21.201.62:43803 requested a full tablet report, sending...
21:04:02.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.324988 22308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 22607
21:04:02.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.325232 22308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-0/wal/instance
21:04:02.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.327365 22350 ts_manager.cc:194] Registered new tserver with Master: 832e52648dfc4f899b53b9ca1cb02855 (127.21.201.1:33341)
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.329568 22308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/wal
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/data
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/logs
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/data/info.pb
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.2:0
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.2
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.2
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:04:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.333372 22761 heartbeater.cc:344] Connected to a master server at 127.21.201.61:41997
21:04:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.333418 22760 heartbeater.cc:344] Connected to a master server at 127.21.201.60:33991
21:04:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.333576 22760 heartbeater.cc:461] Registering TS with master...
21:04:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.333580 22761 heartbeater.cc:461] Registering TS with master...
21:04:02.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.334251 22761 heartbeater.cc:507] Master 127.21.201.61:41997 requested a full tablet report, sending...
21:04:02.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.334272 22760 heartbeater.cc:507] Master 127.21.201.60:33991 requested a full tablet report, sending...
21:04:02.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.335925 22512 ts_manager.cc:194] Registered new tserver with Master: 832e52648dfc4f899b53b9ca1cb02855 (127.21.201.1:33341)
21:04:02.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.335960 22431 ts_manager.cc:194] Registered new tserver with Master: 832e52648dfc4f899b53b9ca1cb02855 (127.21.201.1:33341)
21:04:02.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.337556 22512 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.201.1:59095
21:04:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.479969 22591 catalog_manager.cc:1269] Loaded cluster ID: c74d27e67cf645a686abcaaa16a25183
21:04:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.480079 22591 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403: loading cluster ID for follower catalog manager: success
21:04:02.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.482756 22591 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403: acquiring CA information for follower catalog manager: success
21:04:02.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.484424 22591 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:02.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.672008 22764 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:02.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.672355 22764 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:02.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.672534 22764 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.681495 22764 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.681675 22764 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:04:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.681742 22764 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:04:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.681787 22764 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:04:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.681901 22764 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.201.2
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.693202 22764 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/data
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/wal
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.2:0
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/data/info.pb
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.2
21:04:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.2
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/logs
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:02.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.694828 22764 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.696962 22764 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.706790 22770 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.706856 22769 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:02.707837 22772 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.707947 22764 server_base.cc:1047] running on GCE node
21:04:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.708566 22764 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:02.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.709501 22764 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:02.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.710753 22764 hybrid_clock.cc:648] HybridClock initialized: now 1761685442710675 us; error 42 us; skew 500 ppm
21:04:02.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.711087 22764 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:02.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.713358 22764 webserver.cc:492] Webserver started at http://127.21.201.2:41081/ using document root <none> and password file <none>
21:04:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.714095 22764 fs_manager.cc:362] Metadata directory not provided
21:04:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.714221 22764 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.714597 22764 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:02.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.717471 22764 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/data/instance:
21:04:02.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9347684934434f41b7e9621c9db67659"
21:04:02.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:02 on dist-test-slave-vx1b"
21:04:02.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.718269 22764 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/wal/instance:
21:04:02.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9347684934434f41b7e9621c9db67659"
21:04:02.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:02 on dist-test-slave-vx1b"
21:04:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.723755 22764 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
21:04:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.727229 22778 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.728700 22764 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.728849 22764 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/wal
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9347684934434f41b7e9621c9db67659"
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:02 on dist-test-slave-vx1b"
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.729023 22764 fs_report.cc:389] FS layout report
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/wal
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/wal
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/data/data
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:02.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.746791 22764 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:02.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.747609 22764 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:02.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.747861 22764 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:02.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.750979 22764 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:02.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.751104 22764 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:04:02.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.751223 22764 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:02.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.751307 22764 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:04:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.811746 22764 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.2:41119
21:04:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.811805 22899 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.2:41119 every 8 connection(s)
21:04:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.813109 22764 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/data/info.pb
21:04:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.817206 22900 heartbeater.cc:344] Connected to a master server at 127.21.201.62:43803
21:04:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.817579 22900 heartbeater.cc:461] Registering TS with master...
21:04:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.818498 22900 heartbeater.cc:507] Master 127.21.201.62:43803 requested a full tablet report, sending...
21:04:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.820010 22349 ts_manager.cc:194] Registered new tserver with Master: 9347684934434f41b7e9621c9db67659 (127.21.201.2:41119)
21:04:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.822391 22901 heartbeater.cc:344] Connected to a master server at 127.21.201.60:33991
21:04:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.822554 22901 heartbeater.cc:461] Registering TS with master...
21:04:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.822726 22308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 22764
21:04:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.822902 22308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-1/wal/instance
21:04:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.823396 22901 heartbeater.cc:507] Master 127.21.201.60:33991 requested a full tablet report, sending...
21:04:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.823628 22902 heartbeater.cc:344] Connected to a master server at 127.21.201.61:41997
21:04:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.823768 22902 heartbeater.cc:461] Registering TS with master...
21:04:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.824244 22902 heartbeater.cc:507] Master 127.21.201.61:41997 requested a full tablet report, sending...
21:04:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.824469 22512 ts_manager.cc:194] Registered new tserver with Master: 9347684934434f41b7e9621c9db67659 (127.21.201.2:41119)
21:04:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.825469 22430 ts_manager.cc:194] Registered new tserver with Master: 9347684934434f41b7e9621c9db67659 (127.21.201.2:41119)
21:04:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.825556 22512 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.201.2:35267
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.827124 22308 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/wal
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/data
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/logs
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/data/info.pb
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.3:0
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.3
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.3
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
21:04:02.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.839645 22620 catalog_manager.cc:1269] Loaded cluster ID: c74d27e67cf645a686abcaaa16a25183
21:04:02.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.839776 22620 catalog_manager.cc:1562] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2: loading cluster ID for follower catalog manager: success
21:04:02.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.843811 22620 catalog_manager.cc:1584] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2: acquiring CA information for follower catalog manager: success
21:04:02.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:02.846419 22620 catalog_manager.cc:1612] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:04:03.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.175344 22905 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:04:03.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.175674 22905 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:04:03.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.175851 22905 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:04:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.184157 22905 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:04:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.184336 22905 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
21:04:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.184410 22905 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
21:04:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.184496 22905 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
21:04:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.184633 22905 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.201.3
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.195263 22905 tablet_server_runner.cc:78] Tablet server non-default flags:
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.20:45877
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/data
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/wal
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.3:0
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/data/info.pb
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.3
21:04:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.62:43803,127.21.201.61:41997,127.21.201.60:33991
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.3
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/logs
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Oct 2025 20:43:24 UTC on 24a791456cd2
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8764
21:04:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
21:04:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.197218 22905 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:04:03.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.199179 22905 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:04:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.208689 22911 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.208689 22912 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.210299 22914 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:04:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.210680 22905 server_base.cc:1047] running on GCE node
21:04:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.211381 22905 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:04:03.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.212353 22905 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:04:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.213613 22905 hybrid_clock.cc:648] HybridClock initialized: now 1761685443213551 us; error 40 us; skew 500 ppm
21:04:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.213975 22905 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:04:03.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.216348 22905 webserver.cc:492] Webserver started at http://127.21.201.3:39247/ using document root <none> and password file <none>
21:04:03.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.217136 22905 fs_manager.cc:362] Metadata directory not provided
21:04:03.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.217257 22905 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:04:03.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.217600 22905 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:04:03.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.220082 22905 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/data/instance:
21:04:03.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "246b34da955d418189b9f30b654b0f0d"
21:04:03.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:03 on dist-test-slave-vx1b"
21:04:03.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.220947 22905 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/wal/instance:
21:04:03.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "246b34da955d418189b9f30b654b0f0d"
21:04:03.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:03 on dist-test-slave-vx1b"
21:04:03.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.226468 22905 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
21:04:03.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.229979 22920 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:04:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.231259 22905 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
21:04:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.231422 22905 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/data,/tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/wal
21:04:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "246b34da955d418189b9f30b654b0f0d"
21:04:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-28 21:04:03 on dist-test-slave-vx1b"
21:04:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.231632 22905 fs_report.cc:389] FS layout report
21:04:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:04:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/wal
21:04:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/wal
21:04:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/data/data
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:04:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:04:03.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.263541 22905 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:04:03.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.264458 22905 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:04:03.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.264772 22905 kserver.cc:163] Server-wide thread pool size limit: 3276
21:04:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.267907 22905 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:04:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.268144 22905 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:04:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.268375 22905 ts_tablet_manager.cc:616] Registered 0 tablets
21:04:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.268486 22905 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:04:03.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.330345 22905 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.3:35793
21:04:03.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.330394 23041 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.3:35793 every 8 connection(s)
21:04:03.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.331880 22905 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/data/info.pb
21:04:03.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.334483 23042 heartbeater.cc:344] Connected to a master server at 127.21.201.62:43803
21:04:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.335073 23042 heartbeater.cc:461] Registering TS with master...
21:04:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.336402 22308 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu as pid 22905
21:04:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.336591 22308 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskRxIrUU/test-tmp/mini-kudu-cluster3333122528741142553/ts-2/wal/instance
21:04:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.339057 23042 heartbeater.cc:507] Master 127.21.201.62:43803 requested a full tablet report, sending...
21:04:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.339958 22760 heartbeater.cc:499] Master 127.21.201.60:33991 was elected leader, sending a full tablet report...
21:04:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.341711 22349 ts_manager.cc:194] Registered new tserver with Master: 246b34da955d418189b9f30b654b0f0d (127.21.201.3:35793)
21:04:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.344483 23043 heartbeater.cc:344] Connected to a master server at 127.21.201.60:33991
21:04:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.344610 23043 heartbeater.cc:461] Registering TS with master...
21:04:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.345148 23043 heartbeater.cc:507] Master 127.21.201.60:33991 requested a full tablet report, sending...
21:04:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.346048 22512 ts_manager.cc:194] Registered new tserver with Master: 246b34da955d418189b9f30b654b0f0d (127.21.201.3:35793)
21:04:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.346235 23044 heartbeater.cc:344] Connected to a master server at 127.21.201.61:41997
21:04:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.346374 23044 heartbeater.cc:461] Registering TS with master...
21:04:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.346776 23044 heartbeater.cc:507] Master 127.21.201.61:41997 requested a full tablet report, sending...
21:04:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.347047 22512 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.201.3:44195
21:04:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.347684 22430 ts_manager.cc:194] Registered new tserver with Master: 246b34da955d418189b9f30b654b0f0d (127.21.201.3:35793)
21:04:03.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.362344 22308 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:04:03.364 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:04:03.364 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:04:03.365 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.21.201.62"
port: 43803
}
bound_http_address {
host: "127.21.201.62"
port: 35443
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.21.201.61"
port: 41997
}
bound_http_address {
host: "127.21.201.61"
port: 43775
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.21.201.60"
port: 33991
}
bound_http_address {
host: "127.21.201.60"
port: 44099
}
}
}
21:04:03.365 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:04:03.366 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.21.201.1"
port: 33341
}
bound_http_address {
host: "127.21.201.1"
port: 42113
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.21.201.2"
port: 41119
}
bound_http_address {
host: "127.21.201.2"
port: 41081
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.21.201.3"
port: 35793
}
bound_http_address {
host: "127.21.201.3"
port: 39247
}
}
}
21:04:03.366 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:04:03.368 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:04:03.369 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.201.61:41997(127.21.201.61:41997)] connecting to peer
21:04:03.370 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.201.62:43803(127.21.201.62:43803)] connecting to peer
21:04:03.371 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:03.371 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.201.61:41997(127.21.201.61:41997)] Successfully connected to peer
21:04:03.372 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.21.201.62:43803(127.21.201.62:43803)] Successfully connected to peer
21:04:03.373 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.201.61
21:04:03.373 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.201.62
21:04:03.375 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.201.60:33991(127.21.201.60:33991)] Successfully connected to peer
21:04:03.377 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.21.201.60
21:04:03.387 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x6881bf49, L:/127.0.0.1:48344 - R:/127.21.201.62:43803] using SASL/PLAIN
21:04:03.392 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x176b291f, L:/127.0.0.1:60250 - R:/127.21.201.61:41997] using SASL/PLAIN
21:04:03.393 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x05f98f48, L:/127.0.0.1:53786 - R:/127.21.201.60:33991] using SASL/PLAIN
21:04:03.396 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:04:03.397 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=28), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: OK, deferred=null) after lookup
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.399206 22512 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53786:
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
21:04:03.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
21:04:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
21:04:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
21:04:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.401531 22512 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:03.428 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table ae2b74d917454370b4de2b956c468d8d
21:04:03.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.465966 22827 tablet_service.cc:1505] Processing CreateTablet for tablet bc13161a24a74a85bcaee2ae437b5d4f (DEFAULT_TABLE table=txn_manager_fallback [id=ae2b74d917454370b4de2b956c468d8d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:04:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.466686 22976 tablet_service.cc:1505] Processing CreateTablet for tablet bc13161a24a74a85bcaee2ae437b5d4f (DEFAULT_TABLE table=txn_manager_fallback [id=ae2b74d917454370b4de2b956c468d8d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:04:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.466686 22691 tablet_service.cc:1505] Processing CreateTablet for tablet bc13161a24a74a85bcaee2ae437b5d4f (DEFAULT_TABLE table=txn_manager_fallback [id=ae2b74d917454370b4de2b956c468d8d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
21:04:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.467303 22694 tablet_service.cc:1505] Processing CreateTablet for tablet 50e9782ae70a4cc99051ba0f8c9239bc (DEFAULT_TABLE table=txn_manager_fallback [id=ae2b74d917454370b4de2b956c468d8d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:04:03.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.466962 22977 tablet_service.cc:1505] Processing CreateTablet for tablet 50e9782ae70a4cc99051ba0f8c9239bc (DEFAULT_TABLE table=txn_manager_fallback [id=ae2b74d917454370b4de2b956c468d8d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:04:03.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.467986 22827 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bc13161a24a74a85bcaee2ae437b5d4f. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:03.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.468084 22691 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bc13161a24a74a85bcaee2ae437b5d4f. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:03.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.468231 22976 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bc13161a24a74a85bcaee2ae437b5d4f. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:03.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.468622 22694 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 50e9782ae70a4cc99051ba0f8c9239bc. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:03.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.469748 22977 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 50e9782ae70a4cc99051ba0f8c9239bc. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:03.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.468286 22828 tablet_service.cc:1505] Processing CreateTablet for tablet 50e9782ae70a4cc99051ba0f8c9239bc (DEFAULT_TABLE table=txn_manager_fallback [id=ae2b74d917454370b4de2b956c468d8d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
21:04:03.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.472146 22828 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 50e9782ae70a4cc99051ba0f8c9239bc. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.489935 23068 tablet_bootstrap.cc:492] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659: Bootstrap starting.
21:04:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.490931 23069 tablet_bootstrap.cc:492] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d: Bootstrap starting.
21:04:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.493551 23070 tablet_bootstrap.cc:492] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855: Bootstrap starting.
21:04:03.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.494823 23068 tablet_bootstrap.cc:654] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659: Neither blocks nor log segments found. Creating new log.
21:04:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.494820 23069 tablet_bootstrap.cc:654] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d: Neither blocks nor log segments found. Creating new log.
21:04:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.496212 23069 log.cc:826] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d: Log is configured to *not* fsync() on all Append() calls
21:04:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.496217 23068 log.cc:826] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659: Log is configured to *not* fsync() on all Append() calls
21:04:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.496359 23070 tablet_bootstrap.cc:654] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855: Neither blocks nor log segments found. Creating new log.
21:04:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.497828 23070 log.cc:826] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855: Log is configured to *not* fsync() on all Append() calls
21:04:03.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.499855 23068 tablet_bootstrap.cc:492] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659: No bootstrap required, opened a new log
21:04:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.499855 23069 tablet_bootstrap.cc:492] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d: No bootstrap required, opened a new log
21:04:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.500141 23070 tablet_bootstrap.cc:492] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855: No bootstrap required, opened a new log
21:04:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.500169 23069 ts_tablet_manager.cc:1403] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.004s
21:04:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.500476 23068 ts_tablet_manager.cc:1403] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.005s
21:04:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.500590 23070 ts_tablet_manager.cc:1403] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855: Time spent bootstrapping tablet: real 0.007s user 0.007s sys 0.000s
21:04:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.507688 23068 raft_consensus.cc:359] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.507688 23069 raft_consensus.cc:359] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } }
21:04:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.508112 23069 raft_consensus.cc:385] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.508219 23069 raft_consensus.cc:740] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 246b34da955d418189b9f30b654b0f0d, State: Initialized, Role: FOLLOWER
21:04:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.508385 23068 raft_consensus.cc:385] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.508580 23068 raft_consensus.cc:740] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9347684934434f41b7e9621c9db67659, State: Initialized, Role: FOLLOWER
21:04:03.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.509145 23069 consensus_queue.cc:260] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } }
21:04:03.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.509289 23068 consensus_queue.cc:260] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:03.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.509294 23070 raft_consensus.cc:359] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } }
21:04:03.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.509675 23070 raft_consensus.cc:385] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:03.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.509749 23070 raft_consensus.cc:740] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 832e52648dfc4f899b53b9ca1cb02855, State: Initialized, Role: FOLLOWER
21:04:03.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.510385 23070 consensus_queue.cc:260] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } }
21:04:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.518381 23070 ts_tablet_manager.cc:1434] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855: Time spent starting tablet: real 0.018s user 0.013s sys 0.000s
21:04:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.518384 23069 ts_tablet_manager.cc:1434] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d: Time spent starting tablet: real 0.018s user 0.009s sys 0.009s
21:04:03.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.521142 23043 heartbeater.cc:499] Master 127.21.201.60:33991 was elected leader, sending a full tablet report...
21:04:03.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.522647 23069 tablet_bootstrap.cc:492] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d: Bootstrap starting.
21:04:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.525440 23069 tablet_bootstrap.cc:654] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d: Neither blocks nor log segments found. Creating new log.
21:04:03.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.528595 23070 tablet_bootstrap.cc:492] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855: Bootstrap starting.
21:04:03.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.530454 23070 tablet_bootstrap.cc:654] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855: Neither blocks nor log segments found. Creating new log.
21:04:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.533716 22901 heartbeater.cc:499] Master 127.21.201.60:33991 was elected leader, sending a full tablet report...
21:04:03.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.538460 23068 ts_tablet_manager.cc:1434] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659: Time spent starting tablet: real 0.038s user 0.018s sys 0.010s
21:04:03.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.538741 23069 tablet_bootstrap.cc:492] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d: No bootstrap required, opened a new log
21:04:03.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.539116 23068 tablet_bootstrap.cc:492] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659: Bootstrap starting.
21:04:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.540722 23069 ts_tablet_manager.cc:1403] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d: Time spent bootstrapping tablet: real 0.018s user 0.007s sys 0.003s
21:04:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.541391 23068 tablet_bootstrap.cc:654] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659: Neither blocks nor log segments found. Creating new log.
21:04:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.542186 23069 raft_consensus.cc:359] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.546176 23069 raft_consensus.cc:385] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.546387 23069 raft_consensus.cc:740] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 246b34da955d418189b9f30b654b0f0d, State: Initialized, Role: FOLLOWER
21:04:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.547366 23069 consensus_queue.cc:260] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.549695 23069 ts_tablet_manager.cc:1434] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d: Time spent starting tablet: real 0.009s user 0.001s sys 0.001s
21:04:03.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.550880 23068 tablet_bootstrap.cc:492] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659: No bootstrap required, opened a new log
21:04:03.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.551175 23068 ts_tablet_manager.cc:1403] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659: Time spent bootstrapping tablet: real 0.012s user 0.006s sys 0.004s
21:04:03.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.552197 23068 raft_consensus.cc:359] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } }
21:04:03.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.552534 23068 raft_consensus.cc:385] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:03.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.552721 23068 raft_consensus.cc:740] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9347684934434f41b7e9621c9db67659, State: Initialized, Role: FOLLOWER
21:04:03.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.553079 23068 consensus_queue.cc:260] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } }
21:04:03.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.553696 23068 ts_tablet_manager.cc:1434] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
21:04:03.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.553761 23070 tablet_bootstrap.cc:492] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855: No bootstrap required, opened a new log
21:04:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.553929 23070 ts_tablet_manager.cc:1403] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855: Time spent bootstrapping tablet: real 0.025s user 0.009s sys 0.012s
21:04:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.555127 23070 raft_consensus.cc:359] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.555456 23070 raft_consensus.cc:385] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:03.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.555648 23070 raft_consensus.cc:740] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 832e52648dfc4f899b53b9ca1cb02855, State: Initialized, Role: FOLLOWER
21:04:03.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.556221 23070 consensus_queue.cc:260] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:03.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.556721 23070 ts_tablet_manager.cc:1434] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
21:04:03.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.566195 22903 tablet.cc:2378] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:03.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.577557 22762 tablet.cc:2378] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:03.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:03.589686 23045 tablet.cc:2378] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:04:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.681893 23076 raft_consensus.cc:493] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.682194 23076 raft_consensus.cc:515] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } }
21:04:03.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.684010 23076 leader_election.cc:290] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 246b34da955d418189b9f30b654b0f0d (127.21.201.3:35793), 9347684934434f41b7e9621c9db67659 (127.21.201.2:41119)
21:04:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.691483 22997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "50e9782ae70a4cc99051ba0f8c9239bc" candidate_uuid: "832e52648dfc4f899b53b9ca1cb02855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "246b34da955d418189b9f30b654b0f0d" is_pre_election: true
21:04:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.691481 22852 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "50e9782ae70a4cc99051ba0f8c9239bc" candidate_uuid: "832e52648dfc4f899b53b9ca1cb02855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9347684934434f41b7e9621c9db67659" is_pre_election: true
21:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.691922 22997 raft_consensus.cc:2468] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 832e52648dfc4f899b53b9ca1cb02855 in term 0.
21:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.691957 22852 raft_consensus.cc:2468] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 832e52648dfc4f899b53b9ca1cb02855 in term 0.
21:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.692653 22638 leader_election.cc:304] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 246b34da955d418189b9f30b654b0f0d, 832e52648dfc4f899b53b9ca1cb02855, 9347684934434f41b7e9621c9db67659; no voters:
21:04:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.693095 23076 raft_consensus.cc:2804] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.693256 23076 raft_consensus.cc:493] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.693370 23076 raft_consensus.cc:3060] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Advancing to term 1
21:04:03.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.695461 23076 raft_consensus.cc:515] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } }
21:04:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.696102 23076 leader_election.cc:290] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [CANDIDATE]: Term 1 election: Requested vote from peers 246b34da955d418189b9f30b654b0f0d (127.21.201.3:35793), 9347684934434f41b7e9621c9db67659 (127.21.201.2:41119)
21:04:03.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.696789 22997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "50e9782ae70a4cc99051ba0f8c9239bc" candidate_uuid: "832e52648dfc4f899b53b9ca1cb02855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "246b34da955d418189b9f30b654b0f0d"
21:04:03.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.696822 22852 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "50e9782ae70a4cc99051ba0f8c9239bc" candidate_uuid: "832e52648dfc4f899b53b9ca1cb02855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9347684934434f41b7e9621c9db67659"
21:04:03.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.697000 22997 raft_consensus.cc:3060] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Advancing to term 1
21:04:03.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.697041 22852 raft_consensus.cc:3060] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Advancing to term 1
21:04:03.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.699368 22997 raft_consensus.cc:2468] T 50e9782ae70a4cc99051ba0f8c9239bc P 246b34da955d418189b9f30b654b0f0d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 832e52648dfc4f899b53b9ca1cb02855 in term 1.
21:04:03.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.699378 22852 raft_consensus.cc:2468] T 50e9782ae70a4cc99051ba0f8c9239bc P 9347684934434f41b7e9621c9db67659 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 832e52648dfc4f899b53b9ca1cb02855 in term 1.
21:04:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.699870 22638 leader_election.cc:304] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 246b34da955d418189b9f30b654b0f0d, 832e52648dfc4f899b53b9ca1cb02855, 9347684934434f41b7e9621c9db67659; no voters:
21:04:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.700220 23076 raft_consensus.cc:2804] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 1 FOLLOWER]: Leader election won for term 1
21:04:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.700762 23076 raft_consensus.cc:697] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [term 1 LEADER]: Becoming Leader. State: Replica: 832e52648dfc4f899b53b9ca1cb02855, State: Running, Role: LEADER
21:04:03.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.701596 23076 consensus_queue.cc:237] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } }
21:04:03.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.707679 22512 catalog_manager.cc:5649] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 reported cstate change: term changed from 0 to 1, leader changed from <none> to 832e52648dfc4f899b53b9ca1cb02855 (127.21.201.1). New cstate: current_term: 1 leader_uuid: "832e52648dfc4f899b53b9ca1cb02855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } health_report { overall_health: HEALTHY } } }
21:04:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.800526 23074 raft_consensus.cc:493] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.800746 23074 raft_consensus.cc:515] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:03.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.802627 23074 leader_election.cc:290] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 832e52648dfc4f899b53b9ca1cb02855 (127.21.201.1:33341), 246b34da955d418189b9f30b654b0f0d (127.21.201.3:35793)
21:04:03.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.812227 22714 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "bc13161a24a74a85bcaee2ae437b5d4f" candidate_uuid: "9347684934434f41b7e9621c9db67659" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "832e52648dfc4f899b53b9ca1cb02855" is_pre_election: true
21:04:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.812741 22714 raft_consensus.cc:2468] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9347684934434f41b7e9621c9db67659 in term 0.
21:04:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.812966 22997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "bc13161a24a74a85bcaee2ae437b5d4f" candidate_uuid: "9347684934434f41b7e9621c9db67659" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "246b34da955d418189b9f30b654b0f0d" is_pre_election: true
21:04:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.813393 22997 raft_consensus.cc:2468] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9347684934434f41b7e9621c9db67659 in term 0.
21:04:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.813382 22780 leader_election.cc:304] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 832e52648dfc4f899b53b9ca1cb02855, 9347684934434f41b7e9621c9db67659; no voters:
21:04:03.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.813844 23074 raft_consensus.cc:2804] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:03.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.814023 23074 raft_consensus.cc:493] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:03.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.814129 23074 raft_consensus.cc:3060] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Advancing to term 1
21:04:03.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.817337 23074 raft_consensus.cc:515] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.818117 23074 leader_election.cc:290] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [CANDIDATE]: Term 1 election: Requested vote from peers 832e52648dfc4f899b53b9ca1cb02855 (127.21.201.1:33341), 246b34da955d418189b9f30b654b0f0d (127.21.201.3:35793)
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.818828 22714 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "bc13161a24a74a85bcaee2ae437b5d4f" candidate_uuid: "9347684934434f41b7e9621c9db67659" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "832e52648dfc4f899b53b9ca1cb02855"
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.818821 22997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "bc13161a24a74a85bcaee2ae437b5d4f" candidate_uuid: "9347684934434f41b7e9621c9db67659" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "246b34da955d418189b9f30b654b0f0d"
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.819027 22714 raft_consensus.cc:3060] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Advancing to term 1
21:04:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.819064 22997 raft_consensus.cc:3060] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Advancing to term 1
21:04:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.821357 22714 raft_consensus.cc:2468] T bc13161a24a74a85bcaee2ae437b5d4f P 832e52648dfc4f899b53b9ca1cb02855 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9347684934434f41b7e9621c9db67659 in term 1.
21:04:03.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.821942 22997 raft_consensus.cc:2468] T bc13161a24a74a85bcaee2ae437b5d4f P 246b34da955d418189b9f30b654b0f0d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9347684934434f41b7e9621c9db67659 in term 1.
21:04:03.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.822371 22780 leader_election.cc:304] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 832e52648dfc4f899b53b9ca1cb02855, 9347684934434f41b7e9621c9db67659; no voters:
21:04:03.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.822661 23074 raft_consensus.cc:2804] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 1 FOLLOWER]: Leader election won for term 1
21:04:03.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.823145 23074 raft_consensus.cc:697] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [term 1 LEADER]: Becoming Leader. State: Replica: 9347684934434f41b7e9621c9db67659, State: Running, Role: LEADER
21:04:03.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.823691 23074 consensus_queue.cc:237] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:03.829480 22512 catalog_manager.cc:5649] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9347684934434f41b7e9621c9db67659 (127.21.201.2). New cstate: current_term: 1 leader_uuid: "9347684934434f41b7e9621c9db67659" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } health_report { overall_health: UNKNOWN } } }
21:04:04.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.117274 23076 consensus_queue.cc:1048] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [LEADER]: Connected to new peer: Peer: permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:04.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.126109 23076 consensus_queue.cc:1048] T 50e9782ae70a4cc99051ba0f8c9239bc P 832e52648dfc4f899b53b9ca1cb02855 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:04.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.319741 23074 consensus_queue.cc:1048] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [LEADER]: Connected to new peer: Peer: permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.325999 22512 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53790:
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:04:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:04.327185 22512 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.333413 23107 consensus_queue.cc:1048] T bc13161a24a74a85bcaee2ae437b5d4f P 9347684934434f41b7e9621c9db67659 [LEADER]: Connected to new peer: Peer: permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.360456 22827 tablet_service.cc:1505] Processing CreateTablet for tablet ec4cdbab3522456e9cec06d71b1f13b2 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=a609c765a60549cdaa83a0462f43eeae]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:04:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.360116 22977 tablet_service.cc:1505] Processing CreateTablet for tablet ec4cdbab3522456e9cec06d71b1f13b2 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=a609c765a60549cdaa83a0462f43eeae]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:04:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.360409 22694 tablet_service.cc:1505] Processing CreateTablet for tablet ec4cdbab3522456e9cec06d71b1f13b2 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=a609c765a60549cdaa83a0462f43eeae]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
21:04:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.361310 22977 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec4cdbab3522456e9cec06d71b1f13b2. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.361313 22827 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec4cdbab3522456e9cec06d71b1f13b2. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.361327 22694 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec4cdbab3522456e9cec06d71b1f13b2. 1 dirs total, 0 dirs full, 0 dirs failed
21:04:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.369441 23113 tablet_bootstrap.cc:492] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: Bootstrap starting.
21:04:04.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.371474 23113 tablet_bootstrap.cc:654] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: Neither blocks nor log segments found. Creating new log.
21:04:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.372390 23114 tablet_bootstrap.cc:492] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: Bootstrap starting.
21:04:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.372604 23115 tablet_bootstrap.cc:492] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: Bootstrap starting.
21:04:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.373865 23113 tablet_bootstrap.cc:492] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: No bootstrap required, opened a new log
21:04:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.374010 23113 ts_tablet_manager.cc:1403] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
21:04:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.374365 23114 tablet_bootstrap.cc:654] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: Neither blocks nor log segments found. Creating new log.
21:04:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.374365 23115 tablet_bootstrap.cc:654] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: Neither blocks nor log segments found. Creating new log.
21:04:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.374675 23113 raft_consensus.cc:359] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.374827 23113 raft_consensus.cc:385] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:04.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.374922 23113 raft_consensus.cc:740] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 246b34da955d418189b9f30b654b0f0d, State: Initialized, Role: FOLLOWER
21:04:04.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.375195 23113 consensus_queue.cc:260] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:04.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.375600 23088 tablet_replica.cc:442] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } }
21:04:04.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.375865 23088 tablet_replica.cc:445] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:04.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.379494 23113 ts_tablet_manager.cc:1434] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
21:04:04.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.384050 23115 tablet_bootstrap.cc:492] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: No bootstrap required, opened a new log
21:04:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.384334 23114 tablet_bootstrap.cc:492] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: No bootstrap required, opened a new log
21:04:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.384502 23115 ts_tablet_manager.cc:1403] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: Time spent bootstrapping tablet: real 0.012s user 0.007s sys 0.005s
21:04:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.385809 23115 raft_consensus.cc:359] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.386065 23115 raft_consensus.cc:385] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.386189 23115 raft_consensus.cc:740] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 832e52648dfc4f899b53b9ca1cb02855, State: Initialized, Role: FOLLOWER
21:04:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.386476 23115 consensus_queue.cc:260] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.387084 23115 ts_tablet_manager.cc:1434] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
21:04:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.387730 23114 ts_tablet_manager.cc:1403] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.007s
21:04:04.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.389397 23114 raft_consensus.cc:359] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:04.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.389706 23114 raft_consensus.cc:385] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:04:04.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.389855 23114 raft_consensus.cc:740] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9347684934434f41b7e9621c9db67659, State: Initialized, Role: FOLLOWER
21:04:04.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.390146 23114 consensus_queue.cc:260] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:04.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.390635 23107 tablet_replica.cc:442] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } }
21:04:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.390664 23076 tablet_replica.cc:442] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } }
21:04:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.390885 23076 tablet_replica.cc:445] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.390901 23107 tablet_replica.cc:445] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.390779 23114 ts_tablet_manager.cc:1434] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
21:04:04.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.397352 23107 raft_consensus.cc:493] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:04.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.397533 23107 raft_consensus.cc:515] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:04.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.398212 23107 leader_election.cc:290] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 832e52648dfc4f899b53b9ca1cb02855 (127.21.201.1:33341), 246b34da955d418189b9f30b654b0f0d (127.21.201.3:35793)
21:04:04.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.398782 22714 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "ec4cdbab3522456e9cec06d71b1f13b2" candidate_uuid: "9347684934434f41b7e9621c9db67659" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "832e52648dfc4f899b53b9ca1cb02855" is_pre_election: true
21:04:04.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.398995 22714 raft_consensus.cc:2468] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9347684934434f41b7e9621c9db67659 in term 0.
21:04:04.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.398949 22997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "ec4cdbab3522456e9cec06d71b1f13b2" candidate_uuid: "9347684934434f41b7e9621c9db67659" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "246b34da955d418189b9f30b654b0f0d" is_pre_election: true
21:04:04.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.399211 22997 raft_consensus.cc:2468] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9347684934434f41b7e9621c9db67659 in term 0.
21:04:04.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.399653 22780 leader_election.cc:304] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 832e52648dfc4f899b53b9ca1cb02855, 9347684934434f41b7e9621c9db67659; no voters:
21:04:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.400025 23107 raft_consensus.cc:2804] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:04:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.400121 23107 raft_consensus.cc:493] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.400238 23107 raft_consensus.cc:3060] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 0 FOLLOWER]: Advancing to term 1
21:04:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.402786 23107 raft_consensus.cc:515] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:04.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.403702 23107 leader_election.cc:290] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [CANDIDATE]: Term 1 election: Requested vote from peers 832e52648dfc4f899b53b9ca1cb02855 (127.21.201.1:33341), 246b34da955d418189b9f30b654b0f0d (127.21.201.3:35793)
21:04:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.403808 22714 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "ec4cdbab3522456e9cec06d71b1f13b2" candidate_uuid: "9347684934434f41b7e9621c9db67659" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "832e52648dfc4f899b53b9ca1cb02855"
21:04:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.404019 22714 raft_consensus.cc:3060] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855 [term 0 FOLLOWER]: Advancing to term 1
21:04:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.404196 22997 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "ec4cdbab3522456e9cec06d71b1f13b2" candidate_uuid: "9347684934434f41b7e9621c9db67659" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "246b34da955d418189b9f30b654b0f0d"
21:04:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.404386 22997 raft_consensus.cc:3060] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d [term 0 FOLLOWER]: Advancing to term 1
21:04:04.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.406540 22997 raft_consensus.cc:2468] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9347684934434f41b7e9621c9db67659 in term 1.
21:04:04.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.406971 22781 leader_election.cc:304] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 246b34da955d418189b9f30b654b0f0d, 9347684934434f41b7e9621c9db67659; no voters:
21:04:04.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.407014 22714 raft_consensus.cc:2468] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9347684934434f41b7e9621c9db67659 in term 1.
21:04:04.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.407653 23107 raft_consensus.cc:2804] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 1 FOLLOWER]: Leader election won for term 1
21:04:04.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.407821 23107 raft_consensus.cc:697] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [term 1 LEADER]: Becoming Leader. State: Replica: 9347684934434f41b7e9621c9db67659, State: Running, Role: LEADER
21:04:04.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.408120 23107 consensus_queue.cc:237] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } }
21:04:04.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.409791 23074 tablet_replica.cc:442] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: TxnStatusTablet state changed. Reason: New leader 9347684934434f41b7e9621c9db67659. Latest consensus state: current_term: 1 leader_uuid: "9347684934434f41b7e9621c9db67659" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } }
21:04:04.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.409998 23074 tablet_replica.cc:445] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: This TxnStatusTablet replica's current role is: LEADER
21:04:04.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.411409 22509 catalog_manager.cc:5649] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9347684934434f41b7e9621c9db67659 (127.21.201.2). New cstate: current_term: 1 leader_uuid: "9347684934434f41b7e9621c9db67659" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } health_report { overall_health: UNKNOWN } } }
21:04:04.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.412681 23120 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
21:04:04.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.413116 23120 txn_status_manager.cc:930] Loading transaction status metadata into memory...
21:04:04.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.452332 22714 raft_consensus.cc:1275] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855 [term 1 FOLLOWER]: Refusing update from remote peer 9347684934434f41b7e9621c9db67659: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:04.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.452378 22997 raft_consensus.cc:1275] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d [term 1 FOLLOWER]: Refusing update from remote peer 9347684934434f41b7e9621c9db67659: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:04:04.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.453035 23074 consensus_queue.cc:1048] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [LEADER]: Connected to new peer: Peer: permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:04.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.453327 23107 consensus_queue.cc:1048] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 [LEADER]: Connected to new peer: Peer: permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:04:04.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.461263 23076 tablet_replica.cc:442] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: TxnStatusTablet state changed. Reason: New leader 9347684934434f41b7e9621c9db67659. Latest consensus state: current_term: 1 leader_uuid: "9347684934434f41b7e9621c9db67659" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } }
21:04:04.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.461544 23076 tablet_replica.cc:445] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:04.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.462378 23088 tablet_replica.cc:442] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: TxnStatusTablet state changed. Reason: New leader 9347684934434f41b7e9621c9db67659. Latest consensus state: current_term: 1 leader_uuid: "9347684934434f41b7e9621c9db67659" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } }
21:04:04.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.462604 23088 tablet_replica.cc:445] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:04.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.467058 23094 mvcc.cc:204] Tried to move back new op lower bound from 7215863580469121024 to 7215863580297969664. Current Snapshot: MvccSnapshot[applied={T|T < 7215863580469121024}]
21:04:04.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.476713 23074 tablet_replica.cc:442] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9347684934434f41b7e9621c9db67659" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } }
21:04:04.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.476982 23074 tablet_replica.cc:445] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: This TxnStatusTablet replica's current role is: LEADER
21:04:04.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.482142 23074 tablet_replica.cc:442] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9347684934434f41b7e9621c9db67659" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } }
21:04:04.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.482420 23074 tablet_replica.cc:445] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659: This TxnStatusTablet replica's current role is: LEADER
21:04:04.484 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
21:04:04.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.489109 23088 tablet_replica.cc:442] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9347684934434f41b7e9621c9db67659" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } }
21:04:04.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.489298 23088 tablet_replica.cc:445] T ec4cdbab3522456e9cec06d71b1f13b2 P 246b34da955d418189b9f30b654b0f0d: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:04.491 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table ae2b74d917454370b4de2b956c468d8d
21:04:04.493 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 50e9782ae70a4cc99051ba0f8c9239bc for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
21:04:04.494 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet bc13161a24a74a85bcaee2ae437b5d4f for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
21:04:04.494 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=50e9782ae70a4cc99051ba0f8c9239bc@[246b34da955d418189b9f30b654b0f0d(127.21.201.3:35793),832e52648dfc4f899b53b9ca1cb02855(127.21.201.1:33341)[L],9347684934434f41b7e9621c9db67659(127.21.201.2:41119)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=bc13161a24a74a85bcaee2ae437b5d4f@[246b34da955d418189b9f30b654b0f0d(127.21.201.3:35793),832e52648dfc4f899b53b9ca1cb02855(127.21.201.1:33341),9347684934434f41b7e9621c9db67659(127.21.201.2:41119)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
21:04:04.494 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.21.201.60:33991, [2ms] Sub RPC GetTableLocations: received response from server master-127.21.201.60:33991: OK, deferred=null) after lookup
21:04:04.495 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 832e52648dfc4f899b53b9ca1cb02855(127.21.201.1:33341)] connecting to peer
21:04:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.495251 23076 tablet_replica.cc:442] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9347684934434f41b7e9621c9db67659" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "832e52648dfc4f899b53b9ca1cb02855" member_type: VOTER last_known_addr { host: "127.21.201.1" port: 33341 } } peers { permanent_uuid: "9347684934434f41b7e9621c9db67659" member_type: VOTER last_known_addr { host: "127.21.201.2" port: 41119 } } peers { permanent_uuid: "246b34da955d418189b9f30b654b0f0d" member_type: VOTER last_known_addr { host: "127.21.201.3" port: 35793 } } }
21:04:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.495502 23076 tablet_replica.cc:445] T ec4cdbab3522456e9cec06d71b1f13b2 P 832e52648dfc4f899b53b9ca1cb02855: This TxnStatusTablet replica's current role is: FOLLOWER
21:04:04.500 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 832e52648dfc4f899b53b9ca1cb02855(127.21.201.1:33341)] Successfully connected to peer
21:04:04.520 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd4e8f5f8, L:/127.0.0.1:38594 - R:/127.21.201.1:33341] using TOKEN/null
21:04:04.557 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.201.60:33991
21:04:04.558 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:04:04.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.558724 22308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 22479
21:04:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.560215 22552 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:04.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:04.741705 22308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 22479
21:04:04.763 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.201.60:33991(127.21.201.60:33991)] handling channelInactive
21:04:04.763 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state READY due to: connection closed
21:04:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:04.764387 22417 connection.cc:537] server connection from 127.0.0.1:60234 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:04.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:04.764770 22335 connection.cc:537] server connection from 127.0.0.1:48330 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:04.765 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:04:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:04.837482 22901 heartbeater.cc:646] Failed to heartbeat to 127.21.201.60:33991 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:04.992 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:04.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:04.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:04.994 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.21.201.60:33991(127.21.201.60:33991) for tablet Kudu Master: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.009 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.011 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.011 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.029 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.042313 23136 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 37749401a30c4993b227bfe5d89d3478)
21:04:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.042510 23136 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.042702 23137 raft_consensus.cc:493] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 37749401a30c4993b227bfe5d89d3478)
21:04:05.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.042861 23137 raft_consensus.cc:515] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:05.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.043485 23137 leader_election.cc:290] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5ae294c30f584b8290a509ce13a3d403 (127.21.201.62:43803), 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991)
21:04:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.043962 23136 leader_election.cc:290] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d4eded57c27746c0a0c071bf5e3657c2 (127.21.201.61:41997), 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991)
21:04:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.044456 22439 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5ae294c30f584b8290a509ce13a3d403" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d4eded57c27746c0a0c071bf5e3657c2" is_pre_election: true
21:04:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.044692 22439 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5ae294c30f584b8290a509ce13a3d403 in term 1.
21:04:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.043959 22360 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d4eded57c27746c0a0c071bf5e3657c2" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "5ae294c30f584b8290a509ce13a3d403" is_pre_election: true
21:04:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.045089 22360 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d4eded57c27746c0a0c071bf5e3657c2 in term 1.
21:04:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.045238 22337 leader_election.cc:304] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5ae294c30f584b8290a509ce13a3d403, d4eded57c27746c0a0c071bf5e3657c2; no voters:
21:04:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.045609 22418 leader_election.cc:304] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5ae294c30f584b8290a509ce13a3d403, d4eded57c27746c0a0c071bf5e3657c2; no voters:
21:04:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.045679 23136 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:04:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.045794 23136 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 37749401a30c4993b227bfe5d89d3478)
21:04:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.045868 23137 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:04:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.045892 23136 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 1 FOLLOWER]: Advancing to term 2
21:04:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.045966 23137 raft_consensus.cc:493] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 37749401a30c4993b227bfe5d89d3478)
21:04:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.046043 23137 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 1 FOLLOWER]: Advancing to term 2
21:04:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.046213 22336 leader_election.cc:336] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991): Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.047139 22417 leader_election.cc:336] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991): Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.048257 23136 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.048550 23137 raft_consensus.cc:515] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.048837 23136 leader_election.cc:290] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 2 election: Requested vote from peers d4eded57c27746c0a0c071bf5e3657c2 (127.21.201.61:41997), 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991)
21:04:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.049192 23137 leader_election.cc:290] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [CANDIDATE]: Term 2 election: Requested vote from peers 5ae294c30f584b8290a509ce13a3d403 (127.21.201.62:43803), 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991)
21:04:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.049300 22439 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5ae294c30f584b8290a509ce13a3d403" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d4eded57c27746c0a0c071bf5e3657c2"
21:04:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.049536 22439 raft_consensus.cc:2393] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 5ae294c30f584b8290a509ce13a3d403 in current term 2: Already voted for candidate d4eded57c27746c0a0c071bf5e3657c2 in this term.
21:04:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.049520 22360 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d4eded57c27746c0a0c071bf5e3657c2" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "5ae294c30f584b8290a509ce13a3d403"
21:04:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.049757 22360 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate d4eded57c27746c0a0c071bf5e3657c2 in current term 2: Already voted for candidate 5ae294c30f584b8290a509ce13a3d403 in this term.
21:04:05.049 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.051292 22336 leader_election.cc:336] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991): Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.051440 22336 leader_election.cc:304] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5ae294c30f584b8290a509ce13a3d403; no voters: 37749401a30c4993b227bfe5d89d3478, d4eded57c27746c0a0c071bf5e3657c2
21:04:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.051586 22417 leader_election.cc:336] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991): Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.051692 22417 leader_election.cc:304] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d4eded57c27746c0a0c071bf5e3657c2; no voters: 37749401a30c4993b227bfe5d89d3478, 5ae294c30f584b8290a509ce13a3d403
21:04:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.051765 23136 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:04:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.051996 23137 raft_consensus.cc:2749] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
21:04:05.053 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.070 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.071 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.071 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.071 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.090 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.091 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.092 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.092 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.149 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.150 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.150 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.151 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.169 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.170 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.170 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.170 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.171 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.171 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] sending RPC to server master-127.21.201.60:33991, [4ms] received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [4ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [22ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [40ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [63ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [101ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [102ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [160ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [160ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [180ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [180ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, deferred=Deferred@1889523098(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/48695226@15ccdc7b -> (continuation of Deferred@791501340 after retry RPC after error@1889344616) -> (continuation of Deferred@2124424420 after retry RPC after error@226230351) -> (continuation of Deferred@1212438086 after retry RPC after error@1352826234) -> (continuation of Deferred@794485902 after retry RPC after error@1112143698) -> (continuation of Deferred@685080379 after retry RPC after error@1801424630) -> (continuation of Deferred@324626823 after retry RPC after error@505937881), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/518121423@1d086781 -> (continuation of Deferred@791501340 after retry RPC after error@1889344616) -> (continuation of Deferred@2124424420 after retry RPC after error@226230351) -> (continuation of Deferred@1212438086 after retry RPC after error@1352826234) -> (continuation of Deferred@794485902 after retry RPC after error@1112143698) -> (continuation of Deferred@685080379 after retry RPC after error@1801424630) -> (continuation of Deferred@324626823 after retry RPC after error@505937881)))
21:04:05.172 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] sending RPC to server master-127.21.201.60:33991, [4ms] received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [4ms] delaying RPC due to: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [22ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [40ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [63ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [101ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [102ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [160ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [160ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [180ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [180ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, deferred=Deferred@1889523098(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/48695226@15ccdc7b -> (continuation of Deferred@791501340 after retry RPC after error@1889344616) -> (continuation of Deferred@2124424420 after retry RPC after error@226230351) -> (continuation of Deferred@1212438086 after retry RPC after error@1352826234) -> (continuation of Deferred@794485902 after retry RPC after error@1112143698) -> (continuation of Deferred@685080379 after retry RPC after error@1801424630) -> (continuation of Deferred@324626823 after retry RPC after error@505937881), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/518121423@1d086781 -> (continuation of Deferred@791501340 after retry RPC after error@1889344616) -> (continuation of Deferred@2124424420 after retry RPC after error@226230351) -> (continuation of Deferred@1212438086 after retry RPC after error@1352826234) -> (continuation of Deferred@794485902 after retry RPC after error@1112143698) -> (continuation of Deferred@685080379 after retry RPC after error@1801424630) -> (continuation of Deferred@324626823 after retry RPC after error@505937881)))
21:04:05.189 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.190 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.190 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.191 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.209 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.210 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.210 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.210 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.211 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.229 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.249 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.251 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.251 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.270 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.271 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.271 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.310 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.311 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.311 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.312 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.349 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.351 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.351 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.351 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.429 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.430 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.430 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.431 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.432 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [1ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [1ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [62ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [83ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [162ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [241ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [242ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, deferred=Deferred@17664533(state=PENDING, result=null, callback=(continuation of Deferred@1789149225 after retry RPC after error@1930496562) -> (continuation of Deferred@426635867 after retry RPC after error@1960676929) -> (continuation of Deferred@1018183163 after retry RPC after error@1200349381) -> (continuation of Deferred@822230341 after retry RPC after error@699732642) -> (continuation of Deferred@341290378 after retry RPC after error@214104589) -> (continuation of Deferred@738858693 after retry RPC after error@766190052) -> (continuation of Deferred@79290918 after retry RPC after error@523264558), errback=(continuation of Deferred@1789149225 after retry RPC after error@1930496562) -> (continuation of Deferred@426635867 after retry RPC after error@1960676929) -> (continuation of Deferred@1018183163 after retry RPC after error@1200349381) -> (continuation of Deferred@822230341 after retry RPC after error@699732642) -> (continuation of Deferred@341290378 after retry RPC after error@214104589) -> (continuation of Deferred@738858693 after retry RPC after error@766190052) -> (continuation of Deferred@79290918 after retry RPC after error@523264558)))
21:04:05.432 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [1ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [1ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [21ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [41ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [62ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [83ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [161ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [162ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [241ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [242ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, deferred=Deferred@17664533(state=PENDING, result=null, callback=(continuation of Deferred@1789149225 after retry RPC after error@1930496562) -> (continuation of Deferred@426635867 after retry RPC after error@1960676929) -> (continuation of Deferred@1018183163 after retry RPC after error@1200349381) -> (continuation of Deferred@822230341 after retry RPC after error@699732642) -> (continuation of Deferred@341290378 after retry RPC after error@214104589) -> (continuation of Deferred@738858693 after retry RPC after error@766190052) -> (continuation of Deferred@79290918 after retry RPC after error@523264558), errback=(continuation of Deferred@1789149225 after retry RPC after error@1930496562) -> (continuation of Deferred@426635867 after retry RPC after error@1960676929) -> (continuation of Deferred@1018183163 after retry RPC after error@1200349381) -> (continuation of Deferred@822230341 after retry RPC after error@699732642) -> (continuation of Deferred@341290378 after retry RPC after error@214104589) -> (continuation of Deferred@738858693 after retry RPC after error@766190052) -> (continuation of Deferred@79290918 after retry RPC after error@523264558)))
21:04:05.449 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.451 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.453912 23136 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:04:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.454121 23136 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.454669 23136 leader_election.cc:290] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d4eded57c27746c0a0c071bf5e3657c2 (127.21.201.61:41997), 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991)
21:04:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.455193 22439 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5ae294c30f584b8290a509ce13a3d403" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d4eded57c27746c0a0c071bf5e3657c2" is_pre_election: true
21:04:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.455427 22439 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5ae294c30f584b8290a509ce13a3d403 in term 2.
21:04:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.455890 22337 leader_election.cc:304] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5ae294c30f584b8290a509ce13a3d403, d4eded57c27746c0a0c071bf5e3657c2; no voters:
21:04:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.456216 23136 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 2 FOLLOWER]: Leader pre-election won for term 3
21:04:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.456305 22336 leader_election.cc:336] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991): Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.456328 23136 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:04:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.456516 23136 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 2 FOLLOWER]: Advancing to term 3
21:04:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.458415 23136 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:05.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.459074 23136 leader_election.cc:290] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 3 election: Requested vote from peers d4eded57c27746c0a0c071bf5e3657c2 (127.21.201.61:41997), 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991)
21:04:05.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.459484 22439 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5ae294c30f584b8290a509ce13a3d403" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "d4eded57c27746c0a0c071bf5e3657c2"
21:04:05.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.459692 22439 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 2 FOLLOWER]: Advancing to term 3
21:04:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.460680 22336 leader_election.cc:336] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991): Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.462066 22439 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5ae294c30f584b8290a509ce13a3d403 in term 3.
21:04:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.462445 22337 leader_election.cc:304] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5ae294c30f584b8290a509ce13a3d403, d4eded57c27746c0a0c071bf5e3657c2; no voters: 37749401a30c4993b227bfe5d89d3478
21:04:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.462692 23136 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 3 FOLLOWER]: Leader election won for term 3
21:04:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.463142 23136 raft_consensus.cc:697] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [term 3 LEADER]: Becoming Leader. State: Replica: 5ae294c30f584b8290a509ce13a3d403, State: Running, Role: LEADER
21:04:05.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.463624 23136 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } }
21:04:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.465368 23142 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5ae294c30f584b8290a509ce13a3d403. Latest consensus state: current_term: 3 leader_uuid: "5ae294c30f584b8290a509ce13a3d403" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.465667 23142 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: This master's current role is: LEADER
21:04:05.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.466367 23144 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:04:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.468688 23144 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=a609c765a60549cdaa83a0462f43eeae]
21:04:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.469252 23144 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback [id=ae2b74d917454370b4de2b956c468d8d]
21:04:05.469 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.471 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.471 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.471450 23144 tablet_loader.cc:96] loaded metadata for tablet 50e9782ae70a4cc99051ba0f8c9239bc (table txn_manager_fallback [id=ae2b74d917454370b4de2b956c468d8d])
21:04:05.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.471966 23144 tablet_loader.cc:96] loaded metadata for tablet bc13161a24a74a85bcaee2ae437b5d4f (table txn_manager_fallback [id=ae2b74d917454370b4de2b956c468d8d])
21:04:05.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.472360 23144 tablet_loader.cc:96] loaded metadata for tablet ec4cdbab3522456e9cec06d71b1f13b2 (table kudu_system.kudu_transactions [id=a609c765a60549cdaa83a0462f43eeae])
21:04:05.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.472689 23144 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:04:05.473 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991; will retry
21:04:05.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.473582 23144 catalog_manager.cc:1269] Loaded cluster ID: c74d27e67cf645a686abcaaa16a25183
21:04:05.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.473666 23144 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:04:05.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.474772 23144 catalog_manager.cc:1514] Loading token signing keys...
21:04:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.475562 23144 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403: Loaded TSK: 0
21:04:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.476372 23144 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:04:05.489 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.21.201.60:33991(127.21.201.60:33991)] connecting to peer
21:04:05.490 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.60:33991(127.21.201.60:33991)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.201.60:33991: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991
21:04:05.493 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:04:05.494 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=3, TimeoutTracker(timeout=250, elapsed=44), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [2ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [2ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [24ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.21.201.61:41997,127.21.201.62:43803,127.21.201.60:33991) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.61:41997, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.62:43803, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.201.60:33991, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.60:33991: Network error: Failed to connect to peer master-127.21.201.60:33991(127.21.201.60:33991): Connection refused: /127.21.201.60:33991, [42ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.61:41997: OK, [44ms] Sub RPC ConnectToMaster: received response from server master-127.21.201.62:43803: OK, deferred=Deferred@1556292280(state=PENDING, result=null, callback=(continuation of Deferred@1090390556 after retry RPC after error@619941979) -> (continuation of Deferred@831749095 after retry RPC after error@1954564495), errback=(continuation of Deferred@1090390556 after retry RPC after error@619941979) -> (continuation of Deferred@831749095 after retry RPC after error@1954564495))) after lookup
21:04:05.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.495354 23043 heartbeater.cc:646] Failed to heartbeat to 127.21.201.60:33991 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:05.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.515560 22785 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.025s from last keepalive heartbeat (effective timeout is 1.000s)
21:04:05.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.516979 22833 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.21.201.60:33991)
21:04:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.547279 22336 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111) [suppressed 11 similar messages]
21:04:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.549297 22336 consensus_peers.cc:597] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 -> Peer 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991): Couldn't send request to peer 37749401a30c4993b227bfe5d89d3478. Status: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.550698 22760 heartbeater.cc:646] Failed to heartbeat to 127.21.201.60:33991 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111)
21:04:05.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.552306 23149 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.556589 22349 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:48362:
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
21:04:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
21:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
21:04:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:05.558678 22349 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
21:04:05.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.569214 22439 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [term 3 FOLLOWER]: Refusing update from remote peer 5ae294c30f584b8290a509ce13a3d403: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
21:04:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.569995 23142 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
21:04:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.576732 23171 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5ae294c30f584b8290a509ce13a3d403. Latest consensus state: current_term: 3 leader_uuid: "5ae294c30f584b8290a509ce13a3d403" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.577113 23171 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: This master's current role is: FOLLOWER
21:04:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.578303 23136 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "5ae294c30f584b8290a509ce13a3d403" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.578580 23136 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 [sys.catalog]: This master's current role is: LEADER
21:04:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.579228 23171 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "5ae294c30f584b8290a509ce13a3d403" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5ae294c30f584b8290a509ce13a3d403" member_type: VOTER last_known_addr { host: "127.21.201.62" port: 43803 } } peers { permanent_uuid: "d4eded57c27746c0a0c071bf5e3657c2" member_type: VOTER last_known_addr { host: "127.21.201.61" port: 41997 } } peers { permanent_uuid: "37749401a30c4993b227bfe5d89d3478" member_type: VOTER last_known_addr { host: "127.21.201.60" port: 33991 } } }
21:04:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:05.579389 23171 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4eded57c27746c0a0c071bf5e3657c2 [sys.catalog]: This master's current role is: FOLLOWER
21:04:05.587 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 9347684934434f41b7e9621c9db67659 (127.21.201.2:41119): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1761685444 last_transition_timestamp: 1761685445
21:04:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:06.043459 22336 consensus_peers.cc:597] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 -> Peer 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991): Couldn't send request to peer 37749401a30c4993b227bfe5d89d3478. Status: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:04:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:06.486627 22319 master_runner.cc:186] Error getting master registration for 127.21.201.60:33991: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111),
21:04:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:06.486824 22319 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:04:06.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:06.493750 23042 heartbeater.cc:499] Master 127.21.201.62:43803 was elected leader, sending a full tablet report...
21:04:06.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:06.546713 22336 consensus_peers.cc:597] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 -> Peer 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991): Couldn't send request to peer 37749401a30c4993b227bfe5d89d3478. Status: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
21:04:06.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:06.550994 22759 heartbeater.cc:499] Master 127.21.201.62:43803 was elected leader, sending a full tablet report...
21:04:06.770 [DEBUG - main] (Connection.java:575) [peer master-127.21.201.60:33991(127.21.201.60:33991)] disconnecting while in state TERMINATED
21:04:06.771 [DEBUG - main] (Connection.java:575) [peer 832e52648dfc4f899b53b9ca1cb02855(127.21.201.1:33341)] disconnecting while in state READY
21:04:06.771 [DEBUG - main] (Connection.java:575) [peer master-127.21.201.61:41997(127.21.201.61:41997)] disconnecting while in state READY
21:04:06.771 [DEBUG - main] (Connection.java:575) [peer master-127.21.201.62:43803(127.21.201.62:43803)] disconnecting while in state READY
21:04:06.771 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.201.61:41997(127.21.201.61:41997)] handling channelInactive
21:04:06.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.201.61:41997(127.21.201.61:41997)] cleaning up while in state READY due to: connection closed
21:04:06.772 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:04:06.772 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 832e52648dfc4f899b53b9ca1cb02855(127.21.201.1:33341)] handling channelInactive
21:04:06.772 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 832e52648dfc4f899b53b9ca1cb02855(127.21.201.1:33341)] cleaning up while in state READY due to: connection closed
21:04:06.773 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.21.201.62:43803(127.21.201.62:43803)] handling channelInactive
21:04:06.773 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.201.62:43803(127.21.201.62:43803)] cleaning up while in state READY due to: connection closed
21:04:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:06.773819 22308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 22607
21:04:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:06.781162 22754 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:06.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:06.838680 22900 heartbeater.cc:499] Master 127.21.201.62:43803 was elected leader, sending a full tablet report...
21:04:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:06.886619 22308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 22607
21:04:06.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:06.904946 22780 connection.cc:537] client connection to 127.21.201.1:33341 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:06.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:06.905077 22308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 22764
21:04:06.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:06.905205 22780 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:06.906780 22780 consensus_peers.cc:597] T ec4cdbab3522456e9cec06d71b1f13b2 P 9347684934434f41b7e9621c9db67659 -> Peer 832e52648dfc4f899b53b9ca1cb02855 (127.21.201.1:33341): Couldn't send request to peer 832e52648dfc4f899b53b9ca1cb02855. Status: Network error: Client connection negotiation failed: client connection to 127.21.201.1:33341: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:04:06.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:06.912160 22895 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:07.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:07.031009 22308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 22764
21:04:07.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:07.047241 22308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 22905
21:04:07.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:07.049810 22336 consensus_peers.cc:597] T 00000000000000000000000000000000 P 5ae294c30f584b8290a509ce13a3d403 -> Peer 37749401a30c4993b227bfe5d89d3478 (127.21.201.60:33991): Couldn't send request to peer 37749401a30c4993b227bfe5d89d3478. Status: Network error: Client connection negotiation failed: client connection to 127.21.201.60:33991: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
21:04:07.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:07.053848 23037 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:07.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:07.146507 22308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 22905
21:04:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:07.164404 22308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 22319
21:04:07.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:07.165618 22390 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:07.294644 22308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 22319
21:04:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251028 21:04:07.315418 22418 connection.cc:537] server connection from 127.0.0.1:60218 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:04:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:07.315675 22308 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu pid 22398
21:04:07.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:07.316742 22471 generic_service.cc:196] Checking for leaks (request via RPC)
21:04:07.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251028 21:04:07.423535 22308 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskRxIrUU/build/asan/bin/kudu with pid 22398
21:04:07.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-28T21:04:07Z chronyd exiting
21:04:07.577 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 9347684934434f41b7e9621c9db67659 (127.21.201.2:41119): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1761685444 last_transition_timestamp: 1761685445
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 9347684934434f41b7e9621c9db67659 (127.21.201.2:41119): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1761685444 last_transition_timestamp: 1761685445
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more