15:13:55.149 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:13:55.150 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskG_DMvO/test-tmp
15:13:55.150 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:13:55.150 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskG_DMvO/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:13:55.151 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:13:55.464 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:13:55.464 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:13:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:55.467609 18452 mini_oidc.cc:150] Starting JWKS server
15:13:55.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:55.470700 18452 webserver.cc:492] Webserver started at http://127.0.0.1:37671/ using document root <none> and password file <none>
15:13:55.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:55.471100 18452 mini_oidc.cc:186] Starting OIDC Discovery server
15:13:55.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:55.472770 18452 webserver.cc:492] Webserver started at http://127.0.0.1:41747/ using document root <none> and password file <none>
15:13:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:13:55Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:13:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:13:55Z Disabled control of system clock
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:55.536628 18452 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/wal
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/data
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/logs
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/data/info.pb
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.62:39027
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.62
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37671/jwks.json
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:13:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:13:55.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.833525 18464 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:13:55.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.834069 18464 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:13:55.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.834604 18464 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:13:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.866109 18464 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:13:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.866428 18464 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:13:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.866587 18464 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:13:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.866767 18464 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:13:55.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.866984 18464 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:13:55.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.867187 18464 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:13:55.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.867344 18464 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37671/jwks.json
15:13:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:55.902519 18464 master_runner.cc:386] Master server non-default flags:
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/data
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/wal
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.62:39027
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37671/jwks.json
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/data/info.pb
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.62
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/logs
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:13:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:13:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:55.903825 18464 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:13:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:55.905335 18464 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:13:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.915627 18470 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:13:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:55.916311 18471 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:13:57.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.229825 18472 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1312 milliseconds
15:13:57.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.231395 18473 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:13:57.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.233479 18464 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.317s user 0.424s sys 0.887s
15:13:57.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.233834 18464 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.318s user 0.424s sys 0.887s
15:13:57.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.234313 18464 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:13:57.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.236253 18464 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:13:57.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.239509 18464 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:13:57.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.241080 18464 hybrid_clock.cc:648] HybridClock initialized: now 1765638837240997 us; error 87 us; skew 500 ppm
15:13:57.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.242287 18464 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:13:57.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.251061 18464 webserver.cc:492] Webserver started at http://127.18.5.62:43785/ using document root <none> and password file <none>
15:13:57.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.252564 18464 fs_manager.cc:362] Metadata directory not provided
15:13:57.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.252871 18464 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:13:57.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.253527 18464 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:13:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.260435 18464 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/data/instance:
15:13:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "779e0691761a4e4cb212855ae050a8a0"
15:13:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:13:57 on dist-test-slave-4djg"
15:13:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.262046 18464 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/wal/instance:
15:13:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "779e0691761a4e4cb212855ae050a8a0"
15:13:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:13:57 on dist-test-slave-4djg"
15:13:57.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.272020 18464 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
15:13:57.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.280604 18480 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:13:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.281998 18464 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.282549 18464 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/data,/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/wal
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "779e0691761a4e4cb212855ae050a8a0"
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:13:57 on dist-test-slave-4djg"
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.283033 18464 fs_report.cc:389] FS layout report
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/wal
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/wal
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/data/data
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:13:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:13:57.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:13:57.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:13:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.369068 18464 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:13:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.370605 18464 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:13:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.371008 18464 kserver.cc:163] Server-wide thread pool size limit: 3276
15:13:57.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.450870 18464 rpc_server.cc:307] RPC server started. Bound to: 127.18.5.62:39027
15:13:57.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.450938 18541 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.5.62:39027 every 8 connection(s)
15:13:57.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.453547 18464 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/data/info.pb
15:13:57.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.459074 18542 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:13:57.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.461035 18452 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu as pid 18464
15:13:57.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.461611 18452 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-0/wal/instance
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.466837 18452 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/wal
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/data
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/logs
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/data/info.pb
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.61:34823
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.61
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37671/jwks.json
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:13:57.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.474896 18542 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } has no permanent_uuid. Determining permanent_uuid...
15:13:57.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.499207 18542 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } has no permanent_uuid. Determining permanent_uuid...
15:13:57.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.501223 18483 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.5.61:34823: connect: Connection refused (error 111)
15:13:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.505065 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.61:34823: Network error: Client connection negotiation failed: client connection to 127.18.5.61:34823: connect: Connection refused (error 111)
15:13:57.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.554587 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } attempt: 1
15:13:57.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.558668 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.61:34823: Network error: Client connection negotiation failed: client connection to 127.18.5.61:34823: connect: Connection refused (error 111)
15:13:57.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.627060 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } attempt: 2
15:13:57.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.633548 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.61:34823: Network error: Client connection negotiation failed: client connection to 127.18.5.61:34823: connect: Connection refused (error 111)
15:13:57.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.725005 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } attempt: 3
15:13:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.728762 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.61:34823: Network error: Client connection negotiation failed: client connection to 127.18.5.61:34823: connect: Connection refused (error 111)
15:13:57.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.774968 18544 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:13:57.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.775498 18544 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:13:57.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.775954 18544 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:13:57.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.806600 18544 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:13:57.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.806872 18544 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:13:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.807015 18544 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:13:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.807197 18544 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:13:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.807370 18544 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:13:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.807542 18544 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:13:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.807673 18544 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37671/jwks.json
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.843952 18544 master_runner.cc:386] Master server non-default flags:
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/data
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/wal
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.61:34823
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37671/jwks.json
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/data/info.pb
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.61
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:13:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/logs
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.845304 18544 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:13:57.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.846884 18544 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:13:57.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.856612 18553 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:13:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:57.872217 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } attempt: 4
15:13:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.878495 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.61:34823: Network error: Client connection negotiation failed: client connection to 127.18.5.61:34823: connect: Connection refused (error 111)
15:13:58.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:58.178051 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } attempt: 5
15:13:58.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:58.184415 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.61:34823: Network error: Client connection negotiation failed: client connection to 127.18.5.61:34823: connect: Connection refused (error 111)
15:13:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:58.732066 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } attempt: 6
15:13:58.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:58.746014 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.61:34823: Network error: Client connection negotiation failed: client connection to 127.18.5.61:34823: connect: Connection refused (error 111)
15:13:59.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:57.858345 18554 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:13:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.051292 18556 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:13:59.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.053916 18555 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1192 milliseconds
15:13:59.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.054003 18544 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:13:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.055406 18544 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:13:59.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.057799 18544 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:13:59.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.059149 18544 hybrid_clock.cc:648] HybridClock initialized: now 1765638839059097 us; error 51 us; skew 500 ppm
15:13:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.059964 18544 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:13:59.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.066167 18544 webserver.cc:492] Webserver started at http://127.18.5.61:37559/ using document root <none> and password file <none>
15:13:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.067126 18544 fs_manager.cc:362] Metadata directory not provided
15:13:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.067349 18544 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:13:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.067816 18544 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:13:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.072077 18544 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/data/instance:
15:13:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c577a162257444681464c098a4bc938"
15:13:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:13:59 on dist-test-slave-4djg"
15:13:59.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.073244 18544 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/wal/instance:
15:13:59.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c577a162257444681464c098a4bc938"
15:13:59.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:13:59 on dist-test-slave-4djg"
15:13:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.080588 18544 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
15:13:59.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.086529 18565 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:13:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.087529 18544 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.087877 18544 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/data,/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/wal
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c577a162257444681464c098a4bc938"
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:13:59 on dist-test-slave-4djg"
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.088186 18544 fs_report.cc:389] FS layout report
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/wal
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/wal
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/data/data
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:13:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:13:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:13:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:13:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:13:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:13:59.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.133177 18544 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:13:59.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.134714 18544 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:13:59.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.135195 18544 kserver.cc:163] Server-wide thread pool size limit: 3276
15:13:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.234758 18544 rpc_server.cc:307] RPC server started. Bound to: 127.18.5.61:34823
15:13:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.234825 18626 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.5.61:34823 every 8 connection(s)
15:13:59.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.237432 18544 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/data/info.pb
15:13:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.243229 18627 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:13:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.243548 18452 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu as pid 18544
15:13:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.243980 18452 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-1/wal/instance
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.246294 18452 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/logs
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data/info.pb
15:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.60:39939
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.60
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37671/jwks.json
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:13:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:13:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.256536 18627 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } has no permanent_uuid. Determining permanent_uuid...
15:13:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.281294 18627 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } has no permanent_uuid. Determining permanent_uuid...
15:13:59.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.295686 18627 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } has no permanent_uuid. Determining permanent_uuid...
15:13:59.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.298271 18568 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:13:59.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.302680 18627 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:13:59.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.352126 18627 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 1
15:13:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.356750 18627 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:13:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.425261 18627 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 2
15:13:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.429431 18627 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:13:59.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.520813 18627 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 3
15:13:59.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.525074 18627 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:13:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.566706 18629 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:13:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.567399 18629 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:13:59.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.567958 18629 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:13:59.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.599519 18629 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:13:59.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.599874 18629 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:13:59.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.600133 18629 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:13:59.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.600406 18629 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:13:59.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.600675 18629 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:13:59.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.600955 18629 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:13:59.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.601162 18629 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37671/jwks.json
15:13:59.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.639443 18629 master_runner.cc:386] Master server non-default flags:
15:13:59.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:13:59.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.60:39939
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37671/jwks.json
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data/info.pb
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.60
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/logs
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:13:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:13:59.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:13:59.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:13:59.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:13:59.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:13:59.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:13:59.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.641108 18629 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:13:59.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.642740 18629 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:13:59.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.652693 18639 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:13:59.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.668452 18627 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 4
15:13:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.674610 18627 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:13:59.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.806617 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } attempt: 7
15:13:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.837082 18542 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } has no permanent_uuid. Determining permanent_uuid...
15:13:59.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.848321 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:13:59.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.906980 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 1
15:13:59.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.913800 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:13:59.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.974159 18627 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 5
15:13:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.980790 18627 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:13:59.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:13:59.995374 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 2
15:14:00.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:00.001638 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:14:00.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.087378 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 3
15:14:00.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:00.094645 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:14:00.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.235288 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 4
15:14:00.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:00.243211 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:14:00.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.527041 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 5
15:14:00.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.529453 18627 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 6
15:14:00.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:00.537801 18542 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:14:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:00.542116 18627 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.18.5.60:39939: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:14:00.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:13:59.654492 18640 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:00.861860 18642 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:00.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:00.865495 18641 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1210 milliseconds
15:14:00.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.865581 18629 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:14:00.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.866784 18629 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:14:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.869223 18629 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:14:00.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.870571 18629 hybrid_clock.cc:648] HybridClock initialized: now 1765638840870530 us; error 47 us; skew 500 ppm
15:14:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.871397 18629 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:14:00.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.877350 18629 webserver.cc:492] Webserver started at http://127.18.5.60:42051/ using document root <none> and password file <none>
15:14:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.878367 18629 fs_manager.cc:362] Metadata directory not provided
15:14:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.878583 18629 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:14:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.879071 18629 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:14:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.883417 18629 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data/instance:
15:14:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4"
15:14:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:00 on dist-test-slave-4djg"
15:14:00.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.884496 18629 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal/instance:
15:14:00.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4"
15:14:00.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:00 on dist-test-slave-4djg"
15:14:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.891571 18629 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
15:14:00.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.897231 18655 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:14:00.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.898272 18629 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
15:14:00.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.898597 18629 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data,/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4"
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:00 on dist-test-slave-4djg"
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.898937 18629 fs_report.cc:389] FS layout report
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data/data
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:14:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:14:00.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.954944 18629 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:14:00.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.956414 18629 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:14:00.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:00.956876 18629 kserver.cc:163] Server-wide thread pool size limit: 3276
15:14:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.039254 18629 rpc_server.cc:307] RPC server started. Bound to: 127.18.5.60:39939
15:14:01.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.039326 18716 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.5.60:39939 every 8 connection(s)
15:14:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.042047 18629 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data/info.pb
15:14:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.044209 18452 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu as pid 18629
15:14:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.044745 18452 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal/instance
15:14:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.048460 18717 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:14:01.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.065372 18717 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } has no permanent_uuid. Determining permanent_uuid...
15:14:01.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.088853 18717 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } has no permanent_uuid. Determining permanent_uuid...
15:14:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.090642 18542 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 6
15:14:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.100466 18717 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } has no permanent_uuid. Determining permanent_uuid...
15:14:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.111559 18542 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0: Bootstrap starting.
15:14:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.117101 18717 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: Bootstrap starting.
15:14:01.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.117952 18542 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0: Neither blocks nor log segments found. Creating new log.
15:14:01.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.119688 18542 log.cc:826] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0: Log is configured to *not* fsync() on all Append() calls
15:14:01.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.123090 18717 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: Neither blocks nor log segments found. Creating new log.
15:14:01.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.124788 18542 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0: No bootstrap required, opened a new log
15:14:01.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.125383 18717 log.cc:826] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: Log is configured to *not* fsync() on all Append() calls
15:14:01.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.129921 18717 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: No bootstrap required, opened a new log
15:14:01.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.143208 18542 raft_consensus.cc:359] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:01.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.143939 18542 raft_consensus.cc:385] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:01.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.144233 18542 raft_consensus.cc:740] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 779e0691761a4e4cb212855ae050a8a0, State: Initialized, Role: FOLLOWER
15:14:01.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.145164 18542 consensus_queue.cc:260] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.148952 18728 sys_catalog.cc:455] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:01.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.149780 18728 sys_catalog.cc:458] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: This master's current role is: FOLLOWER
15:14:01.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.150425 18542 sys_catalog.cc:565] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: configured and running, proceeding with master startup.
15:14:01.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.151836 18717 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:01.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.152447 18717 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:01.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.152702 18717 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7833f5c30a5d4d6e8a8278d1c2874cc4, State: Initialized, Role: FOLLOWER
15:14:01.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.153708 18717 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.157076 18732 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.157884 18732 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: This master's current role is: FOLLOWER
15:14:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.159134 18717 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: configured and running, proceeding with master startup.
15:14:01.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.187012 18732 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:14:01.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.187546 18732 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:01.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.190008 18732 leader_election.cc:290] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027), 9c577a162257444681464c098a4bc938 (127.18.5.61:34823)
15:14:01.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.191826 18507 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "779e0691761a4e4cb212855ae050a8a0" is_pre_election: true
15:14:01.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.192698 18507 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7833f5c30a5d4d6e8a8278d1c2874cc4 in term 0.
15:14:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.194283 18658 leader_election.cc:304] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 779e0691761a4e4cb212855ae050a8a0, 7833f5c30a5d4d6e8a8278d1c2874cc4; no voters:
15:14:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.195299 18732 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:14:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.195715 18732 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:14:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.196156 18732 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 0 FOLLOWER]: Advancing to term 1
15:14:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:01.199707 18746 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:14:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:01.200337 18746 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:14:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.204978 18732 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.206979 18732 leader_election.cc:290] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [CANDIDATE]: Term 1 election: Requested vote from peers 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027), 9c577a162257444681464c098a4bc938 (127.18.5.61:34823)
15:14:01.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.208072 18507 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "779e0691761a4e4cb212855ae050a8a0"
15:14:01.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.208609 18507 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 0 FOLLOWER]: Advancing to term 1
15:14:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:01.211867 18661 tablet.cc:2378] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:14:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.216039 18507 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7833f5c30a5d4d6e8a8278d1c2874cc4 in term 1.
15:14:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.222887 18658 leader_election.cc:304] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 779e0691761a4e4cb212855ae050a8a0, 7833f5c30a5d4d6e8a8278d1c2874cc4; no voters:
15:14:01.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.223846 18732 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 1 FOLLOWER]: Leader election won for term 1
15:14:01.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.229537 18732 raft_consensus.cc:697] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 1 LEADER]: Becoming Leader. State: Replica: 7833f5c30a5d4d6e8a8278d1c2874cc4, State: Running, Role: LEADER
15:14:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.185695 18464 master_runner.cc:186] Error getting master registration for 127.18.5.61:34823: OK, instance_id { permanent_uuid: "9c577a162257444681464c098a4bc938" instance_seqno: 1765638839213505 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:14:01.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.235100 18464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:14:01.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.230621 18732 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:01.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.267762 18753 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7833f5c30a5d4d6e8a8278d1c2874cc4. Latest consensus state: current_term: 1 leader_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:01.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.275385 18753 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: This master's current role is: LEADER
15:14:01.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.255425 18629 master_runner.cc:186] Error getting master registration for 127.18.5.61:34823: OK, instance_id { permanent_uuid: "9c577a162257444681464c098a4bc938" instance_seqno: 1765638839213505 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:14:01.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.295595 18629 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:14:01.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:01.296836 18752 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:14:01.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:01.297163 18752 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:14:01.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.297610 18755 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:14:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.304723 18755 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:14:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.324123 18507 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 1 FOLLOWER]: Refusing update from remote peer 7833f5c30a5d4d6e8a8278d1c2874cc4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:14:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.326215 18753 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:14:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.356501 18728 sys_catalog.cc:455] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7833f5c30a5d4d6e8a8278d1c2874cc4. Latest consensus state: current_term: 1 leader_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:01.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.359449 18728 sys_catalog.cc:458] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: This master's current role is: FOLLOWER
15:14:01.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.369222 18728 sys_catalog.cc:455] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:01.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.370489 18728 sys_catalog.cc:458] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: This master's current role is: FOLLOWER
15:14:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.373461 18753 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.374323 18753 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: This master's current role is: LEADER
15:14:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.384997 18755 catalog_manager.cc:1357] Generated new cluster ID: 52f5b3fad7fa49df9ad1c6899b29a152
15:14:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.385308 18755 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:14:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.473510 18755 catalog_manager.cc:1380] Generated new certificate authority record
15:14:01.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.484594 18755 catalog_manager.cc:1514] Loading token signing keys...
15:14:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.520714 18755 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: Generated new TSK 0
15:14:01.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.522253 18755 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.549170 18452 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/wal
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/data
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/logs
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/data/info.pb
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:14:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.1:0
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.5.1
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.1
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
15:14:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:01.569782 18658 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.18.5.61:34823 timed out after 0.362s (SENT)
15:14:01.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.604260 18627 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } attempt: 7
15:14:01.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.663997 18627 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938: Bootstrap starting.
15:14:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.671577 18627 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938: Neither blocks nor log segments found. Creating new log.
15:14:01.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.673995 18627 log.cc:826] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938: Log is configured to *not* fsync() on all Append() calls
15:14:01.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.687052 18627 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938: No bootstrap required, opened a new log
15:14:01.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.718952 18627 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.719959 18627 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.720371 18627 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c577a162257444681464c098a4bc938, State: Initialized, Role: FOLLOWER
15:14:01.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.721349 18627 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:01.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.730064 18767 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.734117 18767 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: This master's current role is: FOLLOWER
15:14:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.733765 18627 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: configured and running, proceeding with master startup.
15:14:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.734426 18588 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c577a162257444681464c098a4bc938" is_pre_election: true
15:14:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.735232 18588 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7833f5c30a5d4d6e8a8278d1c2874cc4 in term 0.
15:14:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.736222 18591 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c577a162257444681464c098a4bc938"
15:14:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.736783 18591 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 0 FOLLOWER]: Advancing to term 1
15:14:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.742415 18589 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c577a162257444681464c098a4bc938"
15:14:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.733954 18592 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c577a162257444681464c098a4bc938" is_pre_election: true
15:14:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:01.744834 18592 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34852 (request call id 1) took 551 ms (client timeout 395 ms). Trace:
15:14:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:01.745229 18592 rpcz_store.cc:269] 1213 15:14:01.193092 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:14:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:14:01.194552 (+ 1460us) service_pool.cc:225] Handling call
15:14:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:14:01.744753 (+550201us) inbound_call.cc:173] Queueing success response
15:14:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:14:01.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.748886 18591 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7833f5c30a5d4d6e8a8278d1c2874cc4 in term 1.
15:14:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.750465 18590 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 1 FOLLOWER]: Refusing update from remote peer 7833f5c30a5d4d6e8a8278d1c2874cc4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:14:01.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:01.757984 18591 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34852 (request call id 2) took 550 ms (client timeout 362 ms). Trace:
15:14:01.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.758383 18732 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:14:01.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:01.758577 18591 rpcz_store.cc:269] 1213 15:14:01.207812 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:14:01.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:14:01.207964 (+ 152us) service_pool.cc:225] Handling call
15:14:01.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:14:01.757917 (+549953us) inbound_call.cc:173] Queueing success response
15:14:01.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:14:01.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.844045 18776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:01.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.846098 18776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: This master's current role is: FOLLOWER
15:14:01.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.845456 18767 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7833f5c30a5d4d6e8a8278d1c2874cc4. Latest consensus state: current_term: 1 leader_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:01.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.846707 18767 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: This master's current role is: FOLLOWER
15:14:01.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.863649 18783 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:14:01.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.945335 18732 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:01.946110 18732 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: This master's current role is: LEADER
15:14:02.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.206393 18746 catalog_manager.cc:1269] Loaded cluster ID: 52f5b3fad7fa49df9ad1c6899b29a152
15:14:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.206888 18746 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0: loading cluster ID for follower catalog manager: success
15:14:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.214139 18746 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0: acquiring CA information for follower catalog manager: success
15:14:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:02.218694 18763 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:14:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:02.219321 18763 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:14:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:02.220359 18763 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:14:02.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.223474 18746 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:14:02.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:02.277200 18763 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:14:02.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:02.278213 18763 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
15:14:02.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:02.278499 18763 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
15:14:02.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:02.278792 18763 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
15:14:02.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:02.279582 18763 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.5.1
15:14:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.342341 18763 tablet_server_runner.cc:78] Tablet server non-default flags:
15:14:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:14:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:14:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/data
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/wal
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.1:0
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/data/info.pb
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.1
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.5.1
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/logs
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:14:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:14:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:14:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:14:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.344018 18763 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:14:02.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.346074 18763 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:14:02.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:02.361404 18789 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:02.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.891948 18783 catalog_manager.cc:1269] Loaded cluster ID: 52f5b3fad7fa49df9ad1c6899b29a152
15:14:02.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.893239 18783 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938: loading cluster ID for follower catalog manager: success
15:14:02.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.901108 18783 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938: acquiring CA information for follower catalog manager: success
15:14:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:02.931411 18783 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:14:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:02.362807 18790 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:03.763120 18788 debug-util.cc:398] Leaking SignalData structure 0x7b0800037660 after lost signal to thread 18763
15:14:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:04.466496 18763 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.105s user 0.706s sys 1.155s
15:14:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:04.472921 18763 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.112s user 0.706s sys 1.155s
15:14:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.494717 18763 server_base.cc:1047] running on GCE node
15:14:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:04.496279 18795 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:04.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.497664 18763 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:14:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.514370 18763 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:14:04.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.516042 18763 hybrid_clock.cc:648] HybridClock initialized: now 1765638844515973 us; error 79 us; skew 500 ppm
15:14:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.517110 18763 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:14:04.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.552433 18763 webserver.cc:492] Webserver started at http://127.18.5.1:44629/ using document root <none> and password file <none>
15:14:04.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.553745 18763 fs_manager.cc:362] Metadata directory not provided
15:14:04.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.553999 18763 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:14:04.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.554510 18763 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:14:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.560223 18763 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/data/instance:
15:14:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ddfc1734f9247beba1bc0f97ca83336"
15:14:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:04 on dist-test-slave-4djg"
15:14:04.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.561635 18763 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/wal/instance:
15:14:04.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ddfc1734f9247beba1bc0f97ca83336"
15:14:04.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:04 on dist-test-slave-4djg"
15:14:04.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.587424 18763 fs_manager.cc:696] Time spent creating directory manager: real 0.025s user 0.006s sys 0.018s
15:14:04.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.607079 18800 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:14:04.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.611217 18763 fs_manager.cc:730] Time spent opening block manager: real 0.019s user 0.009s sys 0.008s
15:14:04.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.611785 18763 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/data,/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/wal
15:14:04.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ddfc1734f9247beba1bc0f97ca83336"
15:14:04.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:04 on dist-test-slave-4djg"
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.612435 18763 fs_report.cc:389] FS layout report
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/wal
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/wal
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/data/data
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:14:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:14:04.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.699136 18763 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:14:04.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.701309 18763 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:14:04.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.701993 18763 kserver.cc:163] Server-wide thread pool size limit: 3276
15:14:04.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.736562 18763 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:14:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.743083 18763 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.007s user 0.000s sys 0.001s
15:14:04.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.743607 18763 ts_tablet_manager.cc:616] Registered 0 tablets
15:14:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:04.743899 18763 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:14:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.034475 18763 rpc_server.cc:307] RPC server started. Bound to: 127.18.5.1:38599
15:14:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.035116 18924 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.5.1:38599 every 8 connection(s)
15:14:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.037214 18763 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/data/info.pb
15:14:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.045296 18452 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu as pid 18763
15:14:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.045733 18452 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-0/wal/instance
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.061372 18452 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/wal
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/data
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/logs
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/data/info.pb
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.2:0
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.5.2
15:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.2
15:14:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:14:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:14:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:14:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:14:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:14:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:14:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:14:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:14:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
15:14:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.070088 18927 heartbeater.cc:344] Connected to a master server at 127.18.5.62:39027
15:14:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.070581 18927 heartbeater.cc:461] Registering TS with master...
15:14:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.071921 18927 heartbeater.cc:507] Master 127.18.5.62:39027 requested a full tablet report, sending...
15:14:05.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.073913 18925 heartbeater.cc:344] Connected to a master server at 127.18.5.60:39939
15:14:05.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.075621 18496 ts_manager.cc:194] Registered new tserver with Master: 6ddfc1734f9247beba1bc0f97ca83336 (127.18.5.1:38599)
15:14:05.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.076822 18925 heartbeater.cc:461] Registering TS with master...
15:14:05.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.077749 18925 heartbeater.cc:507] Master 127.18.5.60:39939 requested a full tablet report, sending...
15:14:05.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.080193 18672 ts_manager.cc:194] Registered new tserver with Master: 6ddfc1734f9247beba1bc0f97ca83336 (127.18.5.1:38599)
15:14:05.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.082981 18672 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.18.5.1:49339
15:14:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.085482 18926 heartbeater.cc:344] Connected to a master server at 127.18.5.61:34823
15:14:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.085796 18926 heartbeater.cc:461] Registering TS with master...
15:14:05.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.086980 18926 heartbeater.cc:507] Master 127.18.5.61:34823 requested a full tablet report, sending...
15:14:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.089732 18582 ts_manager.cc:194] Registered new tserver with Master: 6ddfc1734f9247beba1bc0f97ca83336 (127.18.5.1:38599)
15:14:05.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:05.418270 18930 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:14:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:05.418817 18930 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:14:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:05.419489 18930 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:14:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:05.454751 18930 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:14:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:05.455845 18930 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
15:14:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:05.456159 18930 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
15:14:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:05.456425 18930 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
15:14:05.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:05.457119 18930 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.5.2
15:14:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.509294 18930 tablet_server_runner.cc:78] Tablet server non-default flags:
15:14:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:14:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:14:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:14:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:14:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/data
15:14:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/wal
15:14:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:14:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.2:0
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/data/info.pb
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.2
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.5.2
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/logs
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:14:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:14:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:14:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:14:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:14:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.510653 18930 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:14:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:05.512210 18930 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:14:05.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:05.525035 18938 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:06.087800 18925 heartbeater.cc:499] Master 127.18.5.60:39939 was elected leader, sending a full tablet report...
15:14:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:06.926692 18937 debug-util.cc:398] Leaking SignalData structure 0x7b0800037660 after lost signal to thread 18930
15:14:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:07.342527 18930 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.816s user 0.558s sys 1.158s
15:14:07.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:07.342882 18930 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.817s user 0.558s sys 1.158s
15:14:07.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:05.526564 18939 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:07.345140 18941 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:07.348832 18940 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1817 milliseconds
15:14:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.348860 18930 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:14:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.349956 18930 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:14:07.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.352077 18930 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:14:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.353451 18930 hybrid_clock.cc:648] HybridClock initialized: now 1765638847353420 us; error 34 us; skew 500 ppm
15:14:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.354246 18930 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:14:07.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.360412 18930 webserver.cc:492] Webserver started at http://127.18.5.2:43889/ using document root <none> and password file <none>
15:14:07.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.361266 18930 fs_manager.cc:362] Metadata directory not provided
15:14:07.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.361456 18930 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:14:07.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.361867 18930 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:14:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.366048 18930 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/data/instance:
15:14:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2647dd1730154da29c37a99de6575287"
15:14:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:07 on dist-test-slave-4djg"
15:14:07.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.367175 18930 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/wal/instance:
15:14:07.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2647dd1730154da29c37a99de6575287"
15:14:07.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:07 on dist-test-slave-4djg"
15:14:07.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.374519 18930 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
15:14:07.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.380057 18949 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:14:07.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.381146 18930 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
15:14:07.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.381428 18930 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/data,/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/wal
15:14:07.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2647dd1730154da29c37a99de6575287"
15:14:07.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:07 on dist-test-slave-4djg"
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.381721 18930 fs_report.cc:389] FS layout report
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/wal
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/wal
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/data/data
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:14:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:14:07.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.434221 18930 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:14:07.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.435650 18930 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:14:07.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.436084 18930 kserver.cc:163] Server-wide thread pool size limit: 3276
15:14:07.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.445614 18930 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:14:07.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.445930 18930 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:14:07.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.446282 18930 ts_tablet_manager.cc:616] Registered 0 tablets
15:14:07.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.446471 18930 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:14:07.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.642689 18930 rpc_server.cc:307] RPC server started. Bound to: 127.18.5.2:44243
15:14:07.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.642787 19072 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.5.2:44243 every 8 connection(s)
15:14:07.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.645262 18930 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/data/info.pb
15:14:07.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.655035 18452 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu as pid 18930
15:14:07.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.655694 18452 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-1/wal/instance
15:14:07.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.659993 19075 heartbeater.cc:344] Connected to a master server at 127.18.5.62:39027
15:14:07.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.660491 19075 heartbeater.cc:461] Registering TS with master...
15:14:07.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.661936 19075 heartbeater.cc:507] Master 127.18.5.62:39027 requested a full tablet report, sending...
15:14:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.665766 18496 ts_manager.cc:194] Registered new tserver with Master: 2647dd1730154da29c37a99de6575287 (127.18.5.2:44243)
15:14:07.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.667389 19073 heartbeater.cc:344] Connected to a master server at 127.18.5.60:39939
15:14:07.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.667737 19073 heartbeater.cc:461] Registering TS with master...
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.667855 18452 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/wal
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/data
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/logs
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/data/info.pb
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.3:0
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.5.3
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.3
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
15:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.668608 19073 heartbeater.cc:507] Master 127.18.5.60:39939 requested a full tablet report, sending...
15:14:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.670768 18672 ts_manager.cc:194] Registered new tserver with Master: 2647dd1730154da29c37a99de6575287 (127.18.5.2:44243)
15:14:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.672926 18672 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.18.5.2:36223
15:14:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.676157 19074 heartbeater.cc:344] Connected to a master server at 127.18.5.61:34823
15:14:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.676539 19074 heartbeater.cc:461] Registering TS with master...
15:14:07.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.677276 19074 heartbeater.cc:507] Master 127.18.5.61:34823 requested a full tablet report, sending...
15:14:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:07.679579 18582 ts_manager.cc:194] Registered new tserver with Master: 2647dd1730154da29c37a99de6575287 (127.18.5.2:44243)
15:14:07.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:07.982668 19078 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:14:07.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:07.983125 19078 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:14:07.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:07.983690 19078 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:14:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:08.014780 19078 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:14:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:08.015377 19078 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
15:14:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:08.015558 19078 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
15:14:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:08.015710 19078 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
15:14:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:08.016161 19078 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.5.3
15:14:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:08.060335 19078 tablet_server_runner.cc:78] Tablet server non-default flags:
15:14:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:14:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:14:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:14:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:14:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/data
15:14:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/wal
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.3:0
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/data/info.pb
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.3
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.5.3
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/logs
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:14:08.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:08.061827 19078 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:14:08.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:08.063752 19078 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:14:08.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:08.075922 19085 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:08.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:08.676723 19073 heartbeater.cc:499] Master 127.18.5.60:39939 was elected leader, sending a full tablet report...
15:14:09.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:08.077803 19086 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:09.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:09.337946 19088 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:09.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:09.340802 19087 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1258 milliseconds
15:14:09.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.340960 19078 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:14:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.342162 19078 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:14:09.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.344395 19078 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:14:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.345733 19078 hybrid_clock.cc:648] HybridClock initialized: now 1765638849345707 us; error 38 us; skew 500 ppm
15:14:09.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.346581 19078 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:14:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.353047 19078 webserver.cc:492] Webserver started at http://127.18.5.3:38709/ using document root <none> and password file <none>
15:14:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.354007 19078 fs_manager.cc:362] Metadata directory not provided
15:14:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.354292 19078 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:14:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.354780 19078 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:14:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.359081 19078 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/data/instance:
15:14:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e90a4dbb418e4689b953f8774bcd00a7"
15:14:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:09 on dist-test-slave-4djg"
15:14:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.360155 19078 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/wal/instance:
15:14:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e90a4dbb418e4689b953f8774bcd00a7"
15:14:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:09 on dist-test-slave-4djg"
15:14:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.368052 19078 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
15:14:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.374883 19097 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:14:09.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.376022 19078 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
15:14:09.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.376333 19078 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/data,/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/wal
15:14:09.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e90a4dbb418e4689b953f8774bcd00a7"
15:14:09.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:09 on dist-test-slave-4djg"
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.376626 19078 fs_report.cc:389] FS layout report
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/wal
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/wal
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/data/data
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:14:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:14:09.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.432034 19078 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:14:09.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.433501 19078 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:14:09.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.433929 19078 kserver.cc:163] Server-wide thread pool size limit: 3276
15:14:09.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.444670 19078 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:14:09.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.445048 19078 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
15:14:09.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.445384 19078 ts_tablet_manager.cc:616] Registered 0 tablets
15:14:09.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.445564 19078 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:14:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.641217 19078 rpc_server.cc:307] RPC server started. Bound to: 127.18.5.3:39467
15:14:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.641677 19222 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.5.3:39467 every 8 connection(s)
15:14:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.644765 19078 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/data/info.pb
15:14:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.650277 18452 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu as pid 19078
15:14:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.650787 18452 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/ts-2/wal/instance
15:14:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.668236 19225 heartbeater.cc:344] Connected to a master server at 127.18.5.62:39027
15:14:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.668856 19225 heartbeater.cc:461] Registering TS with master...
15:14:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.670184 19223 heartbeater.cc:344] Connected to a master server at 127.18.5.60:39939
15:14:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.670536 19223 heartbeater.cc:461] Registering TS with master...
15:14:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.671838 19223 heartbeater.cc:507] Master 127.18.5.60:39939 requested a full tablet report, sending...
15:14:09.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.676504 18672 ts_manager.cc:194] Registered new tserver with Master: e90a4dbb418e4689b953f8774bcd00a7 (127.18.5.3:39467)
15:14:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.675148 19225 heartbeater.cc:507] Master 127.18.5.62:39027 requested a full tablet report, sending...
15:14:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.678500 18672 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.18.5.3:53507
15:14:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.679432 19224 heartbeater.cc:344] Connected to a master server at 127.18.5.61:34823
15:14:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.679782 19224 heartbeater.cc:461] Registering TS with master...
15:14:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.680634 19224 heartbeater.cc:507] Master 127.18.5.61:34823 requested a full tablet report, sending...
15:14:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.680699 18496 ts_manager.cc:194] Registered new tserver with Master: e90a4dbb418e4689b953f8774bcd00a7 (127.18.5.3:39467)
15:14:09.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.683173 18582 ts_manager.cc:194] Registered new tserver with Master: e90a4dbb418e4689b953f8774bcd00a7 (127.18.5.3:39467)
15:14:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.694262 18452 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
15:14:09.704 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:14:09.705 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:14:09.706 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.18.5.62"
port: 39027
}
bound_http_address {
host: "127.18.5.62"
port: 43785
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.18.5.61"
port: 34823
}
bound_http_address {
host: "127.18.5.61"
port: 37559
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.18.5.60"
port: 39939
}
bound_http_address {
host: "127.18.5.60"
port: 42051
}
}
}
15:14:09.706 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:14:09.707 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.18.5.1"
port: 38599
}
bound_http_address {
host: "127.18.5.1"
port: 44629
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.18.5.2"
port: 44243
}
bound_http_address {
host: "127.18.5.2"
port: 43889
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.18.5.3"
port: 39467
}
bound_http_address {
host: "127.18.5.3"
port: 38709
}
}
}
15:14:09.707 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:14:09.708 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:14:09.710 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.5.61:34823(127.18.5.61:34823)] connecting to peer
15:14:09.711 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:09.714 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.5.60:39939(127.18.5.60:39939)] connecting to peer
15:14:09.714 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.5.61:34823(127.18.5.61:34823)] Successfully connected to peer
15:14:09.716 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.5.62:39027(127.18.5.62:39027)] Successfully connected to peer
15:14:09.719 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.18.5.61
15:14:09.719 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.18.5.62
15:14:09.724 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.5.60:39939(127.18.5.60:39939)] Successfully connected to peer
15:14:09.735 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x0e2d5633, L:/127.0.0.1:42300 - R:/127.18.5.62:39027] using SASL/PLAIN
15:14:09.737 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.18.5.60
15:14:09.744 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xca228b37, L:/127.0.0.1:37628 - R:/127.18.5.61:34823] using SASL/PLAIN
15:14:09.753 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x15ed8349, L:/127.0.0.1:60266 - R:/127.18.5.60:39939] using SASL/PLAIN
15:14:09.756 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:14:09.757 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=47), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [29ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [46ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, deferred=null) after lookup
15:14:09.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.758908 18672 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60266:
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:09.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:09.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:09.762629 18672 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:14:09.806 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 06c67a41512e4ba790ae5e8991b8e1c8
15:14:09.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.859970 18859 tablet_service.cc:1505] Processing CreateTablet for tablet 42579df6e3fa4a8b99e329c894d358a2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
15:14:09.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.860704 18860 tablet_service.cc:1505] Processing CreateTablet for tablet 68e1d70dacc846b2adf1641511b487a2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
15:14:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.862128 18859 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 42579df6e3fa4a8b99e329c894d358a2. 1 dirs total, 0 dirs full, 0 dirs failed
15:14:09.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.866693 18860 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 68e1d70dacc846b2adf1641511b487a2. 1 dirs total, 0 dirs full, 0 dirs failed
15:14:09.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.868880 19158 tablet_service.cc:1505] Processing CreateTablet for tablet 68e1d70dacc846b2adf1641511b487a2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
15:14:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.870730 19158 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 68e1d70dacc846b2adf1641511b487a2. 1 dirs total, 0 dirs full, 0 dirs failed
15:14:09.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.873382 19157 tablet_service.cc:1505] Processing CreateTablet for tablet 42579df6e3fa4a8b99e329c894d358a2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
15:14:09.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.874548 19157 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 42579df6e3fa4a8b99e329c894d358a2. 1 dirs total, 0 dirs full, 0 dirs failed
15:14:09.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.879416 19008 tablet_service.cc:1505] Processing CreateTablet for tablet 68e1d70dacc846b2adf1641511b487a2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
15:14:09.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.881305 19008 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 68e1d70dacc846b2adf1641511b487a2. 1 dirs total, 0 dirs full, 0 dirs failed
15:14:09.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.884240 19007 tablet_service.cc:1505] Processing CreateTablet for tablet 42579df6e3fa4a8b99e329c894d358a2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
15:14:09.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.885370 19007 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 42579df6e3fa4a8b99e329c894d358a2. 1 dirs total, 0 dirs full, 0 dirs failed
15:14:09.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.919817 19248 tablet_bootstrap.cc:492] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336: Bootstrap starting.
15:14:09.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.923430 19249 tablet_bootstrap.cc:492] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7: Bootstrap starting.
15:14:09.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.929018 19250 tablet_bootstrap.cc:492] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287: Bootstrap starting.
15:14:09.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.930980 19248 tablet_bootstrap.cc:654] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336: Neither blocks nor log segments found. Creating new log.
15:14:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.933286 19248 log.cc:826] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336: Log is configured to *not* fsync() on all Append() calls
15:14:09.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.936486 19249 tablet_bootstrap.cc:654] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7: Neither blocks nor log segments found. Creating new log.
15:14:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.936851 19250 tablet_bootstrap.cc:654] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287: Neither blocks nor log segments found. Creating new log.
15:14:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.938030 19248 tablet_bootstrap.cc:492] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336: No bootstrap required, opened a new log
15:14:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.938545 19248 ts_tablet_manager.cc:1403] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336: Time spent bootstrapping tablet: real 0.019s user 0.014s sys 0.000s
15:14:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.938699 19249 log.cc:826] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7: Log is configured to *not* fsync() on all Append() calls
15:14:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.939951 19250 log.cc:826] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287: Log is configured to *not* fsync() on all Append() calls
15:14:09.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.944657 19249 tablet_bootstrap.cc:492] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7: No bootstrap required, opened a new log
15:14:09.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.945282 19249 ts_tablet_manager.cc:1403] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7: Time spent bootstrapping tablet: real 0.022s user 0.007s sys 0.008s
15:14:09.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.945390 19250 tablet_bootstrap.cc:492] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287: No bootstrap required, opened a new log
15:14:09.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.945822 19250 ts_tablet_manager.cc:1403] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287: Time spent bootstrapping tablet: real 0.017s user 0.006s sys 0.007s
15:14:09.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.965665 19250 raft_consensus.cc:359] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:09.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.966529 19250 raft_consensus.cc:385] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:09.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.966816 19250 raft_consensus.cc:740] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2647dd1730154da29c37a99de6575287, State: Initialized, Role: FOLLOWER
15:14:09.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.966566 19248 raft_consensus.cc:359] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:09.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.967512 19248 raft_consensus.cc:385] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:09.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.967826 19248 raft_consensus.cc:740] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ddfc1734f9247beba1bc0f97ca83336, State: Initialized, Role: FOLLOWER
15:14:09.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.967734 19250 consensus_queue.cc:260] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:09.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.968684 19248 consensus_queue.cc:260] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.972299 19250 ts_tablet_manager.cc:1434] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287: Time spent starting tablet: real 0.026s user 0.022s sys 0.003s
15:14:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.973222 19250 tablet_bootstrap.cc:492] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287: Bootstrap starting.
15:14:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.972704 19249 raft_consensus.cc:359] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } }
15:14:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.973974 19249 raft_consensus.cc:385] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.974396 19249 raft_consensus.cc:740] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e90a4dbb418e4689b953f8774bcd00a7, State: Initialized, Role: FOLLOWER
15:14:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.975724 19249 consensus_queue.cc:260] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } }
15:14:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.981194 19250 tablet_bootstrap.cc:654] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287: Neither blocks nor log segments found. Creating new log.
15:14:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.981902 19248 ts_tablet_manager.cc:1434] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336: Time spent starting tablet: real 0.043s user 0.030s sys 0.008s
15:14:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.982995 19248 tablet_bootstrap.cc:492] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336: Bootstrap starting.
15:14:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.991098 19248 tablet_bootstrap.cc:654] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336: Neither blocks nor log segments found. Creating new log.
15:14:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.992749 19223 heartbeater.cc:499] Master 127.18.5.60:39939 was elected leader, sending a full tablet report...
15:14:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.995065 19249 ts_tablet_manager.cc:1434] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7: Time spent starting tablet: real 0.049s user 0.028s sys 0.012s
15:14:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:09.996037 19249 tablet_bootstrap.cc:492] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7: Bootstrap starting.
15:14:10.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.003333 19249 tablet_bootstrap.cc:654] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7: Neither blocks nor log segments found. Creating new log.
15:14:10.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.005309 19248 tablet_bootstrap.cc:492] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336: No bootstrap required, opened a new log
15:14:10.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.005723 19248 ts_tablet_manager.cc:1403] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336: Time spent bootstrapping tablet: real 0.023s user 0.014s sys 0.004s
15:14:10.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.008843 19248 raft_consensus.cc:359] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } }
15:14:10.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.009706 19248 raft_consensus.cc:385] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:10.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.010061 19248 raft_consensus.cc:740] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ddfc1734f9247beba1bc0f97ca83336, State: Initialized, Role: FOLLOWER
15:14:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.010962 19248 consensus_queue.cc:260] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } }
15:14:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.013863 19248 ts_tablet_manager.cc:1434] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336: Time spent starting tablet: real 0.008s user 0.004s sys 0.004s
15:14:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.015197 19249 tablet_bootstrap.cc:492] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7: No bootstrap required, opened a new log
15:14:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.015586 19249 ts_tablet_manager.cc:1403] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7: Time spent bootstrapping tablet: real 0.020s user 0.017s sys 0.000s
15:14:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.018031 19249 raft_consensus.cc:359] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.018671 19249 raft_consensus.cc:385] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.018958 19249 raft_consensus.cc:740] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e90a4dbb418e4689b953f8774bcd00a7, State: Initialized, Role: FOLLOWER
15:14:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.019622 19249 consensus_queue.cc:260] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.022008 19249 ts_tablet_manager.cc:1434] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7: Time spent starting tablet: real 0.006s user 0.006s sys 0.000s
15:14:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.025770 19250 tablet_bootstrap.cc:492] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287: No bootstrap required, opened a new log
15:14:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.026255 19250 ts_tablet_manager.cc:1403] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287: Time spent bootstrapping tablet: real 0.053s user 0.012s sys 0.034s
15:14:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.031248 19250 raft_consensus.cc:359] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } }
15:14:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.032083 19250 raft_consensus.cc:385] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.032579 19250 raft_consensus.cc:740] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2647dd1730154da29c37a99de6575287, State: Initialized, Role: FOLLOWER
15:14:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.033321 19250 consensus_queue.cc:260] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } }
15:14:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.035450 19250 ts_tablet_manager.cc:1434] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287: Time spent starting tablet: real 0.009s user 0.005s sys 0.000s
15:14:10.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:10.069862 18929 tablet.cc:2378] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:14:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.118976 19257 raft_consensus.cc:493] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:14:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.119501 19257 raft_consensus.cc:515] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } }
15:14:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.121786 19257 leader_election.cc:290] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6ddfc1734f9247beba1bc0f97ca83336 (127.18.5.1:38599), 2647dd1730154da29c37a99de6575287 (127.18.5.2:44243)
15:14:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.130329 18880 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "68e1d70dacc846b2adf1641511b487a2" candidate_uuid: "e90a4dbb418e4689b953f8774bcd00a7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6ddfc1734f9247beba1bc0f97ca83336" is_pre_election: true
15:14:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.131130 18880 raft_consensus.cc:2468] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e90a4dbb418e4689b953f8774bcd00a7 in term 0.
15:14:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.131281 19028 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "68e1d70dacc846b2adf1641511b487a2" candidate_uuid: "e90a4dbb418e4689b953f8774bcd00a7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2647dd1730154da29c37a99de6575287" is_pre_election: true
15:14:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.132164 19028 raft_consensus.cc:2468] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e90a4dbb418e4689b953f8774bcd00a7 in term 0.
15:14:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.132581 19101 leader_election.cc:304] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6ddfc1734f9247beba1bc0f97ca83336, e90a4dbb418e4689b953f8774bcd00a7; no voters:
15:14:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.133392 19257 raft_consensus.cc:2804] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:14:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.133644 19257 raft_consensus.cc:493] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:14:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.133873 19257 raft_consensus.cc:3060] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Advancing to term 1
15:14:10.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.138495 19257 raft_consensus.cc:515] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } }
15:14:10.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.139784 19257 leader_election.cc:290] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [CANDIDATE]: Term 1 election: Requested vote from peers 6ddfc1734f9247beba1bc0f97ca83336 (127.18.5.1:38599), 2647dd1730154da29c37a99de6575287 (127.18.5.2:44243)
15:14:10.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.140595 18880 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "68e1d70dacc846b2adf1641511b487a2" candidate_uuid: "e90a4dbb418e4689b953f8774bcd00a7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6ddfc1734f9247beba1bc0f97ca83336"
15:14:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.140966 18880 raft_consensus.cc:3060] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Advancing to term 1
15:14:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.140803 19028 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "68e1d70dacc846b2adf1641511b487a2" candidate_uuid: "e90a4dbb418e4689b953f8774bcd00a7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2647dd1730154da29c37a99de6575287"
15:14:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.141301 19028 raft_consensus.cc:3060] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Advancing to term 1
15:14:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.146134 18880 raft_consensus.cc:2468] T 68e1d70dacc846b2adf1641511b487a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e90a4dbb418e4689b953f8774bcd00a7 in term 1.
15:14:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.146854 19028 raft_consensus.cc:2468] T 68e1d70dacc846b2adf1641511b487a2 P 2647dd1730154da29c37a99de6575287 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e90a4dbb418e4689b953f8774bcd00a7 in term 1.
15:14:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.147105 19101 leader_election.cc:304] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6ddfc1734f9247beba1bc0f97ca83336, e90a4dbb418e4689b953f8774bcd00a7; no voters:
15:14:10.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.147843 19257 raft_consensus.cc:2804] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 1 FOLLOWER]: Leader election won for term 1
15:14:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.149109 19257 raft_consensus.cc:697] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 1 LEADER]: Becoming Leader. State: Replica: e90a4dbb418e4689b953f8774bcd00a7, State: Running, Role: LEADER
15:14:10.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.149981 19257 consensus_queue.cc:237] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } }
15:14:10.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.163168 18672 catalog_manager.cc:5654] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 reported cstate change: term changed from 0 to 1, leader changed from <none> to e90a4dbb418e4689b953f8774bcd00a7 (127.18.5.3). New cstate: current_term: 1 leader_uuid: "e90a4dbb418e4689b953f8774bcd00a7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } health_report { overall_health: UNKNOWN } } }
15:14:10.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:10.165979 19076 tablet.cc:2378] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:14:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:10.176319 19227 tablet.cc:2378] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:14:10.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.247493 19255 raft_consensus.cc:493] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:14:10.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.247900 19255 raft_consensus.cc:515] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.250247 19255 leader_election.cc:290] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2647dd1730154da29c37a99de6575287 (127.18.5.2:44243), e90a4dbb418e4689b953f8774bcd00a7 (127.18.5.3:39467)
15:14:10.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.262888 19178 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "42579df6e3fa4a8b99e329c894d358a2" candidate_uuid: "6ddfc1734f9247beba1bc0f97ca83336" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e90a4dbb418e4689b953f8774bcd00a7" is_pre_election: true
15:14:10.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.263648 19178 raft_consensus.cc:2468] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6ddfc1734f9247beba1bc0f97ca83336 in term 0.
15:14:10.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.263607 19028 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "42579df6e3fa4a8b99e329c894d358a2" candidate_uuid: "6ddfc1734f9247beba1bc0f97ca83336" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2647dd1730154da29c37a99de6575287" is_pre_election: true
15:14:10.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.264122 19028 raft_consensus.cc:2468] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6ddfc1734f9247beba1bc0f97ca83336 in term 0.
15:14:10.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.264801 18801 leader_election.cc:304] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6ddfc1734f9247beba1bc0f97ca83336, e90a4dbb418e4689b953f8774bcd00a7; no voters:
15:14:10.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.265594 19255 raft_consensus.cc:2804] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:14:10.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.265913 19255 raft_consensus.cc:493] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:14:10.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.266162 19255 raft_consensus.cc:3060] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Advancing to term 1
15:14:10.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.270268 19255 raft_consensus.cc:515] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.271674 19255 leader_election.cc:290] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [CANDIDATE]: Term 1 election: Requested vote from peers 2647dd1730154da29c37a99de6575287 (127.18.5.2:44243), e90a4dbb418e4689b953f8774bcd00a7 (127.18.5.3:39467)
15:14:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.272570 19028 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "42579df6e3fa4a8b99e329c894d358a2" candidate_uuid: "6ddfc1734f9247beba1bc0f97ca83336" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2647dd1730154da29c37a99de6575287"
15:14:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.273031 19028 raft_consensus.cc:3060] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Advancing to term 1
15:14:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.272871 19178 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "42579df6e3fa4a8b99e329c894d358a2" candidate_uuid: "6ddfc1734f9247beba1bc0f97ca83336" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e90a4dbb418e4689b953f8774bcd00a7"
15:14:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.273368 19178 raft_consensus.cc:3060] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Advancing to term 1
15:14:10.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.277087 19028 raft_consensus.cc:2468] T 42579df6e3fa4a8b99e329c894d358a2 P 2647dd1730154da29c37a99de6575287 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6ddfc1734f9247beba1bc0f97ca83336 in term 1.
15:14:10.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.278065 18804 leader_election.cc:304] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2647dd1730154da29c37a99de6575287, 6ddfc1734f9247beba1bc0f97ca83336; no voters:
15:14:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.278759 19255 raft_consensus.cc:2804] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 1 FOLLOWER]: Leader election won for term 1
15:14:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.280339 19178 raft_consensus.cc:2468] T 42579df6e3fa4a8b99e329c894d358a2 P e90a4dbb418e4689b953f8774bcd00a7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6ddfc1734f9247beba1bc0f97ca83336 in term 1.
15:14:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.280433 19255 raft_consensus.cc:697] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [term 1 LEADER]: Becoming Leader. State: Replica: 6ddfc1734f9247beba1bc0f97ca83336, State: Running, Role: LEADER
15:14:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.281931 19255 consensus_queue.cc:237] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.296222 18672 catalog_manager.cc:5654] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6ddfc1734f9247beba1bc0f97ca83336 (127.18.5.1). New cstate: current_term: 1 leader_uuid: "6ddfc1734f9247beba1bc0f97ca83336" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } health_report { overall_health: UNKNOWN } } }
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.428416 18672 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60282:
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
15:14:10.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:10.430965 18672 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:14:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.465832 18860 tablet_service.cc:1505] Processing CreateTablet for tablet 43ca2195a3f348c3bb65b5a918e9ac1f (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7992b04e21b74269a0f400aea0581e71]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
15:14:10.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.467547 18860 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 43ca2195a3f348c3bb65b5a918e9ac1f. 1 dirs total, 0 dirs full, 0 dirs failed
15:14:10.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.467234 19008 tablet_service.cc:1505] Processing CreateTablet for tablet 43ca2195a3f348c3bb65b5a918e9ac1f (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7992b04e21b74269a0f400aea0581e71]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
15:14:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.468411 19008 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 43ca2195a3f348c3bb65b5a918e9ac1f. 1 dirs total, 0 dirs full, 0 dirs failed
15:14:10.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.468993 19158 tablet_service.cc:1505] Processing CreateTablet for tablet 43ca2195a3f348c3bb65b5a918e9ac1f (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=7992b04e21b74269a0f400aea0581e71]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
15:14:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.470562 19158 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 43ca2195a3f348c3bb65b5a918e9ac1f. 1 dirs total, 0 dirs full, 0 dirs failed
15:14:10.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.483208 19250 tablet_bootstrap.cc:492] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: Bootstrap starting.
15:14:10.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.483222 19248 tablet_bootstrap.cc:492] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: Bootstrap starting.
15:14:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.488786 19250 tablet_bootstrap.cc:654] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: Neither blocks nor log segments found. Creating new log.
15:14:10.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.489033 19248 tablet_bootstrap.cc:654] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: Neither blocks nor log segments found. Creating new log.
15:14:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.495165 19249 tablet_bootstrap.cc:492] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: Bootstrap starting.
15:14:10.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.500584 19249 tablet_bootstrap.cc:654] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: Neither blocks nor log segments found. Creating new log.
15:14:10.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.504438 19248 tablet_bootstrap.cc:492] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: No bootstrap required, opened a new log
15:14:10.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.504719 19250 tablet_bootstrap.cc:492] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: No bootstrap required, opened a new log
15:14:10.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.504863 19248 ts_tablet_manager.cc:1403] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: Time spent bootstrapping tablet: real 0.022s user 0.011s sys 0.008s
15:14:10.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.505124 19250 ts_tablet_manager.cc:1403] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: Time spent bootstrapping tablet: real 0.022s user 0.009s sys 0.000s
15:14:10.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.507371 19248 raft_consensus.cc:359] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.508062 19248 raft_consensus.cc:385] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:10.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.508388 19248 raft_consensus.cc:740] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ddfc1734f9247beba1bc0f97ca83336, State: Initialized, Role: FOLLOWER
15:14:10.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.509106 19248 consensus_queue.cc:260] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.510360 19255 tablet_replica.cc:442] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } }
15:14:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.507606 19250 raft_consensus.cc:359] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.510996 19255 tablet_replica.cc:445] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: This TxnStatusTablet replica's current role is: FOLLOWER
15:14:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.511101 19250 raft_consensus.cc:385] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.511559 19250 raft_consensus.cc:740] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2647dd1730154da29c37a99de6575287, State: Initialized, Role: FOLLOWER
15:14:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.512460 19250 consensus_queue.cc:260] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.514793 19250 ts_tablet_manager.cc:1434] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: Time spent starting tablet: real 0.009s user 0.007s sys 0.000s
15:14:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.517444 19248 ts_tablet_manager.cc:1434] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: Time spent starting tablet: real 0.012s user 0.006s sys 0.001s
15:14:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.513972 19254 tablet_replica.cc:442] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } }
15:14:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.520759 19254 tablet_replica.cc:445] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: This TxnStatusTablet replica's current role is: FOLLOWER
15:14:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.526152 19249 tablet_bootstrap.cc:492] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: No bootstrap required, opened a new log
15:14:10.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.526605 19249 ts_tablet_manager.cc:1403] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: Time spent bootstrapping tablet: real 0.032s user 0.012s sys 0.005s
15:14:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.530512 19249 raft_consensus.cc:359] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.531184 19249 raft_consensus.cc:385] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:14:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.531466 19249 raft_consensus.cc:740] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e90a4dbb418e4689b953f8774bcd00a7, State: Initialized, Role: FOLLOWER
15:14:10.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.532226 19249 consensus_queue.cc:260] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.533485 19257 tablet_replica.cc:442] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } }
15:14:10.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.534147 19257 tablet_replica.cc:445] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: This TxnStatusTablet replica's current role is: FOLLOWER
15:14:10.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.534401 19249 ts_tablet_manager.cc:1434] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: Time spent starting tablet: real 0.007s user 0.007s sys 0.000s
15:14:10.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.588604 19254 raft_consensus.cc:493] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:14:10.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.589051 19254 raft_consensus.cc:515] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.591531 19254 leader_election.cc:290] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6ddfc1734f9247beba1bc0f97ca83336 (127.18.5.1:38599), e90a4dbb418e4689b953f8774bcd00a7 (127.18.5.3:39467)
15:14:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.601553 18880 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "43ca2195a3f348c3bb65b5a918e9ac1f" candidate_uuid: "2647dd1730154da29c37a99de6575287" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6ddfc1734f9247beba1bc0f97ca83336" is_pre_election: true
15:14:10.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.601598 19178 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "43ca2195a3f348c3bb65b5a918e9ac1f" candidate_uuid: "2647dd1730154da29c37a99de6575287" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e90a4dbb418e4689b953f8774bcd00a7" is_pre_election: true
15:14:10.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.602048 18880 raft_consensus.cc:2468] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2647dd1730154da29c37a99de6575287 in term 0.
15:14:10.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.602200 19178 raft_consensus.cc:2468] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2647dd1730154da29c37a99de6575287 in term 0.
15:14:10.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.603010 18953 leader_election.cc:304] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2647dd1730154da29c37a99de6575287, 6ddfc1734f9247beba1bc0f97ca83336; no voters:
15:14:10.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.603765 19254 raft_consensus.cc:2804] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:14:10.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.604044 19254 raft_consensus.cc:493] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:14:10.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.604280 19254 raft_consensus.cc:3060] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 0 FOLLOWER]: Advancing to term 1
15:14:10.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.608484 19254 raft_consensus.cc:515] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.609723 19254 leader_election.cc:290] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [CANDIDATE]: Term 1 election: Requested vote from peers 6ddfc1734f9247beba1bc0f97ca83336 (127.18.5.1:38599), e90a4dbb418e4689b953f8774bcd00a7 (127.18.5.3:39467)
15:14:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.610440 18880 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "43ca2195a3f348c3bb65b5a918e9ac1f" candidate_uuid: "2647dd1730154da29c37a99de6575287" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6ddfc1734f9247beba1bc0f97ca83336"
15:14:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.610646 19178 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "43ca2195a3f348c3bb65b5a918e9ac1f" candidate_uuid: "2647dd1730154da29c37a99de6575287" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e90a4dbb418e4689b953f8774bcd00a7"
15:14:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.610843 18880 raft_consensus.cc:3060] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336 [term 0 FOLLOWER]: Advancing to term 1
15:14:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.611105 19178 raft_consensus.cc:3060] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7 [term 0 FOLLOWER]: Advancing to term 1
15:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.614910 19178 raft_consensus.cc:2468] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2647dd1730154da29c37a99de6575287 in term 1.
15:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.614916 18880 raft_consensus.cc:2468] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2647dd1730154da29c37a99de6575287 in term 1.
15:14:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.615707 18953 leader_election.cc:304] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2647dd1730154da29c37a99de6575287, 6ddfc1734f9247beba1bc0f97ca83336; no voters:
15:14:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.616375 19254 raft_consensus.cc:2804] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 1 FOLLOWER]: Leader election won for term 1
15:14:10.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.618606 19254 raft_consensus.cc:697] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [term 1 LEADER]: Becoming Leader. State: Replica: 2647dd1730154da29c37a99de6575287, State: Running, Role: LEADER
15:14:10.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.619519 19254 consensus_queue.cc:237] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } }
15:14:10.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.630056 19284 tablet_replica.cc:442] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: TxnStatusTablet state changed. Reason: New leader 2647dd1730154da29c37a99de6575287. Latest consensus state: current_term: 1 leader_uuid: "2647dd1730154da29c37a99de6575287" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } }
15:14:10.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.631384 19284 tablet_replica.cc:445] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: This TxnStatusTablet replica's current role is: LEADER
15:14:10.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.634717 19286 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
15:14:10.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.634644 18671 catalog_manager.cc:5654] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2647dd1730154da29c37a99de6575287 (127.18.5.2). New cstate: current_term: 1 leader_uuid: "2647dd1730154da29c37a99de6575287" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } health_report { overall_health: UNKNOWN } } }
15:14:10.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.635247 19286 txn_status_manager.cc:930] Loading transaction status metadata into memory...
15:14:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.656378 19257 consensus_queue.cc:1048] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:14:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.676401 19255 consensus_queue.cc:1048] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:14:10.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.710105 19257 consensus_queue.cc:1048] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:14:10.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.737607 19265 consensus_queue.cc:1048] T 42579df6e3fa4a8b99e329c894d358a2 P 6ddfc1734f9247beba1bc0f97ca83336 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
15:14:10.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:10.754536 19101 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.18.5.1:38599, user_credentials={real_user=slave}} blocked reactor thread for 50419.3us
15:14:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.778040 19177 raft_consensus.cc:1275] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7 [term 1 FOLLOWER]: Refusing update from remote peer 2647dd1730154da29c37a99de6575287: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:14:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.779397 18879 raft_consensus.cc:1275] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336 [term 1 FOLLOWER]: Refusing update from remote peer 2647dd1730154da29c37a99de6575287: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:14:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.793287 19284 consensus_queue.cc:1048] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:14:10.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.795730 19284 consensus_queue.cc:1048] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:14:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.814841 19255 tablet_replica.cc:442] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: TxnStatusTablet state changed. Reason: New leader 2647dd1730154da29c37a99de6575287. Latest consensus state: current_term: 1 leader_uuid: "2647dd1730154da29c37a99de6575287" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } }
15:14:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.815553 19255 tablet_replica.cc:445] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: This TxnStatusTablet replica's current role is: FOLLOWER
15:14:10.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.820576 19292 mvcc.cc:204] Tried to move back new op lower bound from 7232056732766437376 to 7232056732147068928. Current Snapshot: MvccSnapshot[applied={T|T < 7232056732766437376}]
15:14:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.824164 19294 mvcc.cc:204] Tried to move back new op lower bound from 7232056732766437376 to 7232056732147068928. Current Snapshot: MvccSnapshot[applied={T|T < 7232056732766437376}]
15:14:10.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.829407 19254 tablet_replica.cc:442] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2647dd1730154da29c37a99de6575287" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } }
15:14:10.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.830042 19254 tablet_replica.cc:445] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: This TxnStatusTablet replica's current role is: LEADER
15:14:10.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.831462 19257 tablet_replica.cc:442] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: TxnStatusTablet state changed. Reason: New leader 2647dd1730154da29c37a99de6575287. Latest consensus state: current_term: 1 leader_uuid: "2647dd1730154da29c37a99de6575287" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } }
15:14:10.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.832129 19257 tablet_replica.cc:445] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: This TxnStatusTablet replica's current role is: FOLLOWER
15:14:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.844930 19254 tablet_replica.cc:442] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2647dd1730154da29c37a99de6575287" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } }
15:14:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.845670 19254 tablet_replica.cc:445] T 43ca2195a3f348c3bb65b5a918e9ac1f P 2647dd1730154da29c37a99de6575287: This TxnStatusTablet replica's current role is: LEADER
15:14:10.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.848001 19255 tablet_replica.cc:442] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2647dd1730154da29c37a99de6575287" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } }
15:14:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.848794 19255 tablet_replica.cc:445] T 43ca2195a3f348c3bb65b5a918e9ac1f P 6ddfc1734f9247beba1bc0f97ca83336: This TxnStatusTablet replica's current role is: FOLLOWER
15:14:10.864 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
15:14:10.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.866565 19257 tablet_replica.cc:442] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2647dd1730154da29c37a99de6575287" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" member_type: VOTER last_known_addr { host: "127.18.5.1" port: 38599 } } peers { permanent_uuid: "2647dd1730154da29c37a99de6575287" member_type: VOTER last_known_addr { host: "127.18.5.2" port: 44243 } } peers { permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" member_type: VOTER last_known_addr { host: "127.18.5.3" port: 39467 } } }
15:14:10.867 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 06c67a41512e4ba790ae5e8991b8e1c8
15:14:10.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.867182 19257 tablet_replica.cc:445] T 43ca2195a3f348c3bb65b5a918e9ac1f P e90a4dbb418e4689b953f8774bcd00a7: This TxnStatusTablet replica's current role is: FOLLOWER
15:14:10.868 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.5.60:39939
15:14:10.868 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:14:10.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:10.869506 18452 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu with pid 18629
15:14:10.902 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.5.60:39939(127.18.5.60:39939)] handling channelInactive
15:14:10.903 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.60:39939(127.18.5.60:39939)] cleaning up while in state READY due to: connection closed
15:14:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:10.902719 19100 connection.cc:537] client connection to 127.18.5.60:39939 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:14:10.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:10.903991 19223 heartbeater.cc:646] Failed to heartbeat to 127.18.5.60:39939 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:14:10.905 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:14:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.156931 19304 raft_consensus.cc:493] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 7833f5c30a5d4d6e8a8278d1c2874cc4)
15:14:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.157359 19304 raft_consensus.cc:515] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:11.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.159027 19304 leader_election.cc:290] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9c577a162257444681464c098a4bc938 (127.18.5.61:34823), 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939)
15:14:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.160006 18590 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "779e0691761a4e4cb212855ae050a8a0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "9c577a162257444681464c098a4bc938" is_pre_election: true
15:14:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.160759 18590 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 779e0691761a4e4cb212855ae050a8a0 in term 1.
15:14:11.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.163218 19306 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 7833f5c30a5d4d6e8a8278d1c2874cc4)
15:14:11.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.163797 19306 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:11.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.165170 18483 leader_election.cc:304] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 779e0691761a4e4cb212855ae050a8a0, 9c577a162257444681464c098a4bc938; no voters:
15:14:11.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.166122 19304 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 1 FOLLOWER]: Leader pre-election won for term 2
15:14:11.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:11.166199 18483 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111) [suppressed 12 similar messages]
15:14:11.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.166545 19304 raft_consensus.cc:493] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 7833f5c30a5d4d6e8a8278d1c2874cc4)
15:14:11.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.166930 19304 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 1 FOLLOWER]: Advancing to term 2
15:14:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:11.173388 18568 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111) [suppressed 6 similar messages]
15:14:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.174333 19306 leader_election.cc:290] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027), 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939)
15:14:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.177742 19304 raft_consensus.cc:515] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.178815 18506 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9c577a162257444681464c098a4bc938" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "779e0691761a4e4cb212855ae050a8a0" is_pre_election: true
15:14:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.179685 18506 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 9c577a162257444681464c098a4bc938 in current term 2: Already voted for candidate 779e0691761a4e4cb212855ae050a8a0 in this term.
15:14:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.180034 19304 leader_election.cc:290] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [CANDIDATE]: Term 2 election: Requested vote from peers 9c577a162257444681464c098a4bc938 (127.18.5.61:34823), 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939)
15:14:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:11.180601 18483 leader_election.cc:336] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:14:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.183302 18590 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "779e0691761a4e4cb212855ae050a8a0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "9c577a162257444681464c098a4bc938"
15:14:11.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.183811 18590 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 1 FOLLOWER]: Advancing to term 2
15:14:11.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:11.183982 18568 leader_election.cc:336] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:14:11.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.184880 18568 leader_election.cc:304] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9c577a162257444681464c098a4bc938; no voters: 779e0691761a4e4cb212855ae050a8a0, 7833f5c30a5d4d6e8a8278d1c2874cc4
15:14:11.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:11.187104 18483 leader_election.cc:336] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:14:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.190270 18590 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 779e0691761a4e4cb212855ae050a8a0 in term 2.
15:14:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.190656 19306 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
15:14:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.191284 18483 leader_election.cc:304] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 779e0691761a4e4cb212855ae050a8a0, 9c577a162257444681464c098a4bc938; no voters: 7833f5c30a5d4d6e8a8278d1c2874cc4
15:14:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.192091 19304 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 2 FOLLOWER]: Leader election won for term 2
15:14:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.193389 19304 raft_consensus.cc:697] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [term 2 LEADER]: Becoming Leader. State: Replica: 779e0691761a4e4cb212855ae050a8a0, State: Running, Role: LEADER
15:14:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.194590 19304 consensus_queue.cc:237] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.197597 19310 sys_catalog.cc:455] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 779e0691761a4e4cb212855ae050a8a0. Latest consensus state: current_term: 2 leader_uuid: "779e0691761a4e4cb212855ae050a8a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.198518 19310 sys_catalog.cc:458] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: This master's current role is: LEADER
15:14:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.200104 19311 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:14:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.206946 19311 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8]
15:14:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.208073 19311 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=7992b04e21b74269a0f400aea0581e71]
15:14:11.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.212462 19311 tablet_loader.cc:96] loaded metadata for tablet 42579df6e3fa4a8b99e329c894d358a2 (table txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8])
15:14:11.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.213347 19311 tablet_loader.cc:96] loaded metadata for tablet 43ca2195a3f348c3bb65b5a918e9ac1f (table kudu_system.kudu_transactions [id=7992b04e21b74269a0f400aea0581e71])
15:14:11.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.214208 19311 tablet_loader.cc:96] loaded metadata for tablet 68e1d70dacc846b2adf1641511b487a2 (table txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8])
15:14:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.215081 19311 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:14:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.217689 19311 catalog_manager.cc:1269] Loaded cluster ID: 52f5b3fad7fa49df9ad1c6899b29a152
15:14:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.217873 19311 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:14:11.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.221160 19311 catalog_manager.cc:1514] Loading token signing keys...
15:14:11.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.223875 19311 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0: Loaded TSK: 0
15:14:11.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.224766 19311 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:14:11.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:11.283421 18925 heartbeater.cc:646] Failed to heartbeat to 127.18.5.60:39939 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:14:11.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.289005 18590 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 2 FOLLOWER]: Refusing update from remote peer 779e0691761a4e4cb212855ae050a8a0: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
15:14:11.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.290395 19310 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
15:14:11.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.294397 19306 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 779e0691761a4e4cb212855ae050a8a0. Latest consensus state: current_term: 2 leader_uuid: "779e0691761a4e4cb212855ae050a8a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:11.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.295372 19306 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: This master's current role is: FOLLOWER
15:14:11.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:11.304153 18483 consensus_peers.cc:597] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 -> Peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Couldn't send request to peer 7833f5c30a5d4d6e8a8278d1c2874cc4. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:14:11.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.314702 19306 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "779e0691761a4e4cb212855ae050a8a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:11.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.315639 19306 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: This master's current role is: FOLLOWER
15:14:11.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.319953 19304 sys_catalog.cc:455] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "779e0691761a4e4cb212855ae050a8a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:11.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.320444 19304 sys_catalog.cc:458] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 [sys.catalog]: This master's current role is: LEADER
15:14:11.371 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.60:39939(127.18.5.60:39939)] connecting to peer
15:14:11.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.5.60:39939(127.18.5.60:39939): Connection refused: /127.18.5.60:39939
15:14:11.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.5.60:39939(127.18.5.60:39939)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.60:39939(127.18.5.60:39939): Connection refused: /127.18.5.60:39939
15:14:11.373 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.18.5.60:39939(127.18.5.60:39939) for tablet Kudu Master: Failed to connect to peer master-127.18.5.60:39939(127.18.5.60:39939): Connection refused: /127.18.5.60:39939
15:14:11.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.60:39939(127.18.5.60:39939)] connecting to peer
15:14:11.391 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.5.60:39939(127.18.5.60:39939): Connection refused: /127.18.5.60:39939
15:14:11.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.60:39939(127.18.5.60:39939)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.60:39939(127.18.5.60:39939): Connection refused: /127.18.5.60:39939
15:14:11.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.60:39939: Failed to connect to peer master-127.18.5.60:39939(127.18.5.60:39939): Connection refused: /127.18.5.60:39939
15:14:11.395 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:14:11.395 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=24), Traces: [0ms] sending RPC to server master-127.18.5.60:39939, [2ms] received response from server master-127.18.5.60:39939: Network error: Failed to connect to peer master-127.18.5.60:39939(127.18.5.60:39939): Connection refused: /127.18.5.60:39939, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.18.5.60:39939(127.18.5.60:39939): Connection refused: /127.18.5.60:39939, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [21ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: Network error: Failed to connect to peer master-127.18.5.60:39939(127.18.5.60:39939): Connection refused: /127.18.5.60:39939, [21ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: OK, deferred=Deferred@378880017(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1486068290@5ad96677, errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/879066594@29ce751c)) after lookup
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:11.435590 18495 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42322:
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
15:14:11.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:11.440651 18495 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:14:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:11.809885 18483 consensus_peers.cc:597] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 -> Peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Couldn't send request to peer 7833f5c30a5d4d6e8a8278d1c2874cc4. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
15:14:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:11.861558 19073 heartbeater.cc:646] Failed to heartbeat to 127.18.5.60:39939 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111)
15:14:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:12.312955 19225 heartbeater.cc:499] Master 127.18.5.62:39027 was elected leader, sending a full tablet report...
15:14:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:12.314386 18927 heartbeater.cc:499] Master 127.18.5.62:39027 was elected leader, sending a full tablet report...
15:14:12.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:12.319636 18483 consensus_peers.cc:597] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 -> Peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Couldn't send request to peer 7833f5c30a5d4d6e8a8278d1c2874cc4. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
15:14:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:12.858623 18483 consensus_peers.cc:597] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 -> Peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Couldn't send request to peer 7833f5c30a5d4d6e8a8278d1c2874cc4. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
15:14:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:12.862411 19075 heartbeater.cc:499] Master 127.18.5.62:39027 was elected leader, sending a full tablet report...
15:14:12.913 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 68e1d70dacc846b2adf1641511b487a2 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
15:14:12.913 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 42579df6e3fa4a8b99e329c894d358a2 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
15:14:12.913 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=68e1d70dacc846b2adf1641511b487a2@[2647dd1730154da29c37a99de6575287(127.18.5.2:44243),6ddfc1734f9247beba1bc0f97ca83336(127.18.5.1:38599),e90a4dbb418e4689b953f8774bcd00a7(127.18.5.3:39467)[L]]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=42579df6e3fa4a8b99e329c894d358a2@[2647dd1730154da29c37a99de6575287(127.18.5.2:44243),6ddfc1734f9247beba1bc0f97ca83336(127.18.5.1:38599)[L],e90a4dbb418e4689b953f8774bcd00a7(127.18.5.3:39467)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
15:14:12.913 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=3), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.18.5.62:39027, [3ms] Sub RPC GetTableLocations: received response from server master-127.18.5.62:39027: OK, deferred=null) after lookup
15:14:12.914 [DEBUG - kudu-nio-1] (Connection.java:748) [peer e90a4dbb418e4689b953f8774bcd00a7(127.18.5.3:39467)] connecting to peer
15:14:12.914 [DEBUG - kudu-nio-1] (Connection.java:757) [peer e90a4dbb418e4689b953f8774bcd00a7(127.18.5.3:39467)] Successfully connected to peer
15:14:12.929 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa41edc1f, L:/127.0.0.1:41060 - R:/127.18.5.3:39467] using TOKEN/null
15:14:12.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:12.934703 19113 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.18.5.60:39939)
15:14:13.000 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.5.60:39939
15:14:13.001 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:14:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:13.002529 18452 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:14:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu
15:14:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/logs
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data/info.pb
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.60:39939
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.60
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42051
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37671/jwks.json
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
15:14:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
15:14:13.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.315564 19356 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:14:13.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.316236 19356 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:14:13.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.316802 19356 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:14:13.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.350854 19356 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:14:13.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.351213 19356 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
15:14:13.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.351464 19356 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:14:13.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.351717 19356 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:14:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.351948 19356 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:14:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.352152 19356 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:14:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.352299 19356 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37671/jwks.json
15:14:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:13.388427 19356 master_runner.cc:386] Master server non-default flags:
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.5.20:44959
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.5.62:39027,127.18.5.61:34823,127.18.5.60:39939
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.5.60:39939
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37671/jwks.json
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data/info.pb
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.5.60
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42051
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/logs
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:14:13.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:14:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:13.389811 19356 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:14:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:13.391516 19356 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:14:13.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.404109 19364 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:13.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.404605 18483 consensus_peers.cc:597] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 -> Peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Couldn't send request to peer 7833f5c30a5d4d6e8a8278d1c2874cc4. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
15:14:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.915827 18483 consensus_peers.cc:597] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 -> Peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Couldn't send request to peer 7833f5c30a5d4d6e8a8278d1c2874cc4. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
15:14:14.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:14.463752 18483 consensus_peers.cc:597] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 -> Peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Couldn't send request to peer 7833f5c30a5d4d6e8a8278d1c2874cc4. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
15:14:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:13.405193 19365 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:14.839861 19367 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:14:14.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:14.843406 19366 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1436 milliseconds
15:14:14.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.843500 19356 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:14:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.844849 19356 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:14:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.847412 19356 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:14:14.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.848788 19356 hybrid_clock.cc:648] HybridClock initialized: now 1765638854848744 us; error 46 us; skew 500 ppm
15:14:14.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.849666 19356 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:14:14.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.856015 19356 webserver.cc:492] Webserver started at http://127.18.5.60:42051/ using document root <none> and password file <none>
15:14:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.857175 19356 fs_manager.cc:362] Metadata directory not provided
15:14:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.857410 19356 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:14:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.866137 19356 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
15:14:14.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.870713 19377 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:14:14.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.871752 19356 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
15:14:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.872052 19356 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data,/tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal
15:14:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4"
15:14:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:14:00 on dist-test-slave-4djg"
15:14:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.874073 19356 fs_report.cc:389] FS layout report
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/wal
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data/data
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:14:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:14:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.931435 19356 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:14:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.932955 19356 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:14:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:14.933420 19356 kserver.cc:163] Server-wide thread pool size limit: 3276
15:14:14.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:14.942235 18483 consensus_peers.cc:597] T 00000000000000000000000000000000 P 779e0691761a4e4cb212855ae050a8a0 -> Peer 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939): Couldn't send request to peer 7833f5c30a5d4d6e8a8278d1c2874cc4. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.60:39939: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
15:14:15.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.025028 19356 rpc_server.cc:307] RPC server started. Bound to: 127.18.5.60:39939
15:14:15.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.025115 19439 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.5.60:39939 every 8 connection(s)
15:14:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.027765 19356 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskG_DMvO/test-tmp/mini-kudu-cluster6468618580160551155/master-2/data/info.pb
15:14:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.036175 18452 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu as pid 19356
15:14:15.036 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:14:15.036 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.5.62:39027
15:14:15.037 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:14:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.037395 18452 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu with pid 18464
15:14:15.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.042337 19440 sys_catalog.cc:263] Verifying existing consensus state
15:14:15.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.050124 19440 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: Bootstrap starting.
15:14:15.071 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.5.62:39027(127.18.5.62:39027)] handling channelInactive
15:14:15.071 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state READY due to: connection closed
15:14:15.072 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:14:15.090 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.092 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.092 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.18.5.62:39027(127.18.5.62:39027) for tablet Kudu Master: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.100148 19440 log.cc:826] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: Log is configured to *not* fsync() on all Append() calls
15:14:15.110 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.111 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.60:39939(127.18.5.60:39939)] connecting to peer
15:14:15.112 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.5.60:39939(127.18.5.60:39939)] Successfully connected to peer
15:14:15.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.122 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.18.5.60
15:14:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.136651 19440 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:14:15.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.137494 19440 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: Bootstrap complete.
15:14:15.138 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4302279d, L:/127.0.0.1:32842 - R:/127.18.5.60:39939] using SASL/PLAIN
15:14:15.147 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.60:39939: Catalog manager is not initialized. State: Starting
15:14:15.147 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.151 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.152 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.152 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.154 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.60:39939: Catalog manager is not initialized. State: Starting
15:14:15.154 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.161123 19440 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:15.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.163471 19440 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7833f5c30a5d4d6e8a8278d1c2874cc4, State: Initialized, Role: FOLLOWER
15:14:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.164325 19440 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.167034 19444 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.167860 19444 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: This master's current role is: FOLLOWER
15:14:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.168431 19440 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: configured and running, proceeding with master startup.
15:14:15.171 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.172 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.175 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.60:39939: Catalog manager is not initialized. State: Starting
15:14:15.175 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.191 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.196 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.204128 19455 catalog_manager.cc:1269] Loaded cluster ID: 52f5b3fad7fa49df9ad1c6899b29a152
15:14:15.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.204571 19455 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: loading cluster ID for follower catalog manager: success
15:14:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.212340 19455 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: acquiring CA information for follower catalog manager: success
15:14:15.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.218271 19455 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:14:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.200297 19356 master_runner.cc:186] Error getting master registration for 127.18.5.62:39027: Network error: Client connection negotiation failed: client connection to 127.18.5.62:39027: connect: Connection refused (error 111),
15:14:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.223290 19356 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:14:15.230 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.233 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.233 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.235 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.286721 19458 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 779e0691761a4e4cb212855ae050a8a0)
15:14:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.287218 19458 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.289299 19458 leader_election.cc:290] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027), 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939)
15:14:15.290 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.294 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.295 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.296 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=204), Traces: [0ms] sending RPC to server master-127.18.5.62:39027, [2ms] received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [22ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [23ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [56ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [57ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [62ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [63ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [64ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [83ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [84ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [85ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [85ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [102ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [104ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [106ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [107ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [143ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [144ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [203ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [203ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [205ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [205ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, deferred=Deferred@1119411595(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1486068290@41a7aa02 -> (continuation of Deferred@2076454091 after retry RPC after error@17969967) -> (continuation of Deferred@96662019 after retry RPC after error@2034876163) -> (continuation of Deferred@598828693 after retry RPC after error@858432135) -> (continuation of Deferred@560602123 after retry RPC after error@367988354) -> (continuation of Deferred@1036142436 after retry RPC after error@1536881857), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/879066594@2a6121fd -> (continuation of Deferred@2076454091 after retry RPC after error@17969967) -> (continuation of Deferred@96662019 after retry RPC after error@2034876163) -> (continuation of Deferred@598828693 after retry RPC after error@858432135) -> (continuation of Deferred@560602123 after retry RPC after error@367988354) -> (continuation of Deferred@1036142436 after retry RPC after error@1536881857)))
15:14:15.296 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=204), Traces: [0ms] sending RPC to server master-127.18.5.62:39027, [2ms] received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [22ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [23ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [56ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [57ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [62ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [63ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [64ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [83ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [84ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [85ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [85ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [102ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [104ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [106ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [107ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [143ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [144ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [203ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [203ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [205ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [205ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, deferred=Deferred@1119411595(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1486068290@41a7aa02 -> (continuation of Deferred@2076454091 after retry RPC after error@17969967) -> (continuation of Deferred@96662019 after retry RPC after error@2034876163) -> (continuation of Deferred@598828693 after retry RPC after error@858432135) -> (continuation of Deferred@560602123 after retry RPC after error@367988354) -> (continuation of Deferred@1036142436 after retry RPC after error@1536881857), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/879066594@2a6121fd -> (continuation of Deferred@2076454091 after retry RPC after error@17969967) -> (continuation of Deferred@96662019 after retry RPC after error@2034876163) -> (continuation of Deferred@598828693 after retry RPC after error@858432135) -> (continuation of Deferred@560602123 after retry RPC after error@367988354) -> (continuation of Deferred@1036142436 after retry RPC after error@1536881857)))
15:14:15.310 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.312 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.312 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.312 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.317 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:15.324079 18568 leader_election.cc:336] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027): Network error: Client connection negotiation failed: client connection to 127.18.5.62:39027: connect: Connection refused (error 111)
15:14:15.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.328282 18925 heartbeater.cc:344] Connected to a master server at 127.18.5.60:39939
15:14:15.330 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.335 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.335 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.335 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.337 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.338457 19395 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6ddfc1734f9247beba1bc0f97ca83336" instance_seqno: 1765638844993736) as {username='slave'} at 127.18.5.1:45971; Asking this server to re-register.
15:14:15.350 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.350764 19405 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9c577a162257444681464c098a4bc938" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" is_pre_election: true
15:14:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.351985 19405 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c577a162257444681464c098a4bc938 in term 1.
15:14:15.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.355196 18568 leader_election.cc:304] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7833f5c30a5d4d6e8a8278d1c2874cc4, 9c577a162257444681464c098a4bc938; no voters: 779e0691761a4e4cb212855ae050a8a0
15:14:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.356002 19458 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 2 FOLLOWER]: Leader pre-election won for term 3
15:14:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.356357 19458 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 779e0691761a4e4cb212855ae050a8a0)
15:14:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.356685 19458 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 2 FOLLOWER]: Advancing to term 3
15:14:15.364 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.365295 18925 heartbeater.cc:461] Registering TS with master...
15:14:15.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.369045 19458 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:15.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.371012 19395 ts_manager.cc:194] Registered new tserver with Master: 6ddfc1734f9247beba1bc0f97ca83336 (127.18.5.1:38599)
15:14:15.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.372961 19223 heartbeater.cc:344] Connected to a master server at 127.18.5.60:39939
15:14:15.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.375170 19458 leader_election.cc:290] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [CANDIDATE]: Term 3 election: Requested vote from peers 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027), 7833f5c30a5d4d6e8a8278d1c2874cc4 (127.18.5.60:39939)
15:14:15.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.376582 19405 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9c577a162257444681464c098a4bc938" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4"
15:14:15.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.377238 19405 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 1 FOLLOWER]: Advancing to term 3
15:14:15.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.377817 19395 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e90a4dbb418e4689b953f8774bcd00a7" instance_seqno: 1765638849608686) as {username='slave'} at 127.18.5.3:42361; Asking this server to re-register.
15:14:15.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.379878 19223 heartbeater.cc:461] Registering TS with master...
15:14:15.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:15.386019 18568 leader_election.cc:336] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027): Network error: Client connection negotiation failed: client connection to 127.18.5.62:39027: connect: Connection refused (error 111)
15:14:15.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.386675 19405 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c577a162257444681464c098a4bc938 in term 3.
15:14:15.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.387845 18568 leader_election.cc:304] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7833f5c30a5d4d6e8a8278d1c2874cc4, 9c577a162257444681464c098a4bc938; no voters: 779e0691761a4e4cb212855ae050a8a0
15:14:15.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.389046 19458 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 3 FOLLOWER]: Leader election won for term 3
15:14:15.390 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.392 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.392411 19458 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [term 3 LEADER]: Becoming Leader. State: Replica: 9c577a162257444681464c098a4bc938, State: Running, Role: LEADER
15:14:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.393126 19395 ts_manager.cc:194] Registered new tserver with Master: e90a4dbb418e4689b953f8774bcd00a7 (127.18.5.3:39467)
15:14:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.393368 19458 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } }
15:14:15.404 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.404124 19468 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9c577a162257444681464c098a4bc938. Latest consensus state: current_term: 3 leader_uuid: "9c577a162257444681464c098a4bc938" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:15.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.404882 19468 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: This master's current role is: LEADER
15:14:15.411 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.412 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.413 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.416 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.425233 19470 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:14:15.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.437707 19470 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8]
15:14:15.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.439924 19470 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=7992b04e21b74269a0f400aea0581e71]
15:14:15.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.448760 19470 tablet_loader.cc:96] loaded metadata for tablet 42579df6e3fa4a8b99e329c894d358a2 (table txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8])
15:14:15.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.449776 19470 tablet_loader.cc:96] loaded metadata for tablet 43ca2195a3f348c3bb65b5a918e9ac1f (table kudu_system.kudu_transactions [id=7992b04e21b74269a0f400aea0581e71])
15:14:15.450 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.451064 19470 tablet_loader.cc:96] loaded metadata for tablet 68e1d70dacc846b2adf1641511b487a2 (table txn_manager_fallback_rolling [id=06c67a41512e4ba790ae5e8991b8e1c8])
15:14:15.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.452317 19470 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:14:15.455 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.455 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.455 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.457 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939; will retry
15:14:15.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.457464 19470 catalog_manager.cc:1269] Loaded cluster ID: 52f5b3fad7fa49df9ad1c6899b29a152
15:14:15.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.457666 19470 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:14:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.461005 19470 catalog_manager.cc:1514] Loading token signing keys...
15:14:15.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.463917 19470 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938: Loaded TSK: 0
15:14:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.464879 19470 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:14:15.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:15.477839 18568 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 -> Peer 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027): Couldn't send request to peer 779e0691761a4e4cb212855ae050a8a0. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.62:39027: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:14:15.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.519702 19405 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [term 3 FOLLOWER]: Refusing update from remote peer 9c577a162257444681464c098a4bc938: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
15:14:15.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.521191 19468 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
15:14:15.531 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.5.62:39027(127.18.5.62:39027)] connecting to peer
15:14:15.534 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.534 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.62:39027(127.18.5.62:39027)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.534 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.5.62:39027: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027
15:14:15.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.532390 19444 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9c577a162257444681464c098a4bc938. Latest consensus state: current_term: 3 leader_uuid: "9c577a162257444681464c098a4bc938" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:15.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.533089 19444 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: This master's current role is: FOLLOWER
15:14:15.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.533617 19444 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "9c577a162257444681464c098a4bc938" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.539551 19474 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "9c577a162257444681464c098a4bc938" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.540290 19444 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: This master's current role is: FOLLOWER
15:14:15.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.540436 19474 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7833f5c30a5d4d6e8a8278d1c2874cc4 [sys.catalog]: This master's current role is: FOLLOWER
15:14:15.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.545539 19468 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "9c577a162257444681464c098a4bc938" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "779e0691761a4e4cb212855ae050a8a0" member_type: VOTER last_known_addr { host: "127.18.5.62" port: 39027 } } peers { permanent_uuid: "9c577a162257444681464c098a4bc938" member_type: VOTER last_known_addr { host: "127.18.5.61" port: 34823 } } peers { permanent_uuid: "7833f5c30a5d4d6e8a8278d1c2874cc4" member_type: VOTER last_known_addr { host: "127.18.5.60" port: 39939 } } }
15:14:15.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.546164 19468 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 [sys.catalog]: This master's current role is: LEADER
15:14:15.546 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:14:15.547 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=236), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [2ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [4ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [7ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [7ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [25ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [26ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [27ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [43ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [44ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [54ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [54ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [82ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [84ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [94ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [102ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [103ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [106ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [145ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [146ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [146ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [147ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [224ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [228ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [235ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, deferred=Deferred@1800134981(state=PENDING, result=null, callback=(continuation of Deferred@107943165 after retry RPC after error@1753177269) -> (continuation of Deferred@948527221 after retry RPC after error@217903080) -> (continuation of Deferred@1925896822 after retry RPC after error@1614981742) -> (continuation of Deferred@331085019 after retry RPC after error@2027145963) -> (continuation of Deferred@1550759661 after retry RPC after error@696936148) -> (continuation of Deferred@1708872210 after retry RPC after error@62474448), errback=(continuation of Deferred@107943165 after retry RPC after error@1753177269) -> (continuation of Deferred@948527221 after retry RPC after error@217903080) -> (continuation of Deferred@1925896822 after retry RPC after error@1614981742) -> (continuation of Deferred@331085019 after retry RPC after error@2027145963) -> (continuation of Deferred@1550759661 after retry RPC after error@696936148) -> (continuation of Deferred@1708872210 after retry RPC after error@62474448))) after lookup
15:14:15.564 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=254), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [2ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [4ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [7ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [7ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [25ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [26ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [27ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [43ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [44ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [54ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [54ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [82ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [84ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [94ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [102ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [103ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [106ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [145ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [146ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [146ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [147ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [224ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [228ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [235ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [237ms] sending RPC to server master-127.18.5.61:34823, [254ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1800134981(state=PENDING, result=null, callback=(continuation of Deferred@107943165 after retry RPC after error@1753177269) -> (continuation of Deferred@948527221 after retry RPC after error@217903080) -> (continuation of Deferred@1925896822 after retry RPC after error@1614981742) -> (continuation of Deferred@331085019 after retry RPC after error@2027145963) -> (continuation of Deferred@1550759661 after retry RPC after error@696936148) -> (continuation of Deferred@1708872210 after retry RPC after error@62474448) -> (continuation of Deferred@717840276 after retry RPC@1343168735), errback=(continuation of Deferred@107943165 after retry RPC after error@1753177269) -> (continuation of Deferred@948527221 after retry RPC after error@217903080) -> (continuation of Deferred@1925896822 after retry RPC after error@1614981742) -> (continuation of Deferred@331085019 after retry RPC after error@2027145963) -> (continuation of Deferred@1550759661 after retry RPC after error@696936148) -> (continuation of Deferred@1708872210 after retry RPC after error@62474448) -> (continuation of Deferred@717840276 after retry RPC@1343168735)))
15:14:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:15.561586 18612 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:37628 (request call id 15) took 13 ms (client timeout 13 ms). Trace:
15:14:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:15.562062 18612 rpcz_store.cc:269] 1213 15:14:15.548113 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:14:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:14:15.548285 (+ 172us) service_pool.cc:225] Handling call
15:14:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:14:15.561352 (+ 13067us) inbound_call.cc:173] Queueing success response
15:14:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":505,"init.queue_time_us":2162,"thread_start_us":8263,"threads_started":5}
15:14:15.565 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=254), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [2ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [4ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [7ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [7ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [25ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [26ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [27ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [43ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [44ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [54ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [54ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [82ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [84ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [94ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [102ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [103ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [106ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [145ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [146ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [146ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [147ms] delaying RPC due to: Service unavailable: Master config (127.18.5.61:34823,127.18.5.62:39027,127.18.5.60:39939) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.61:34823, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.62:39027, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.5.60:39939, [224ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.62:39027: Network error: Failed to connect to peer master-127.18.5.62:39027(127.18.5.62:39027): Connection refused: /127.18.5.62:39027, [228ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.60:39939: OK, [235ms] Sub RPC ConnectToMaster: received response from server master-127.18.5.61:34823: OK, [237ms] sending RPC to server master-127.18.5.61:34823, [254ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1800134981(state=PENDING, result=null, callback=(continuation of Deferred@107943165 after retry RPC after error@1753177269) -> (continuation of Deferred@948527221 after retry RPC after error@217903080) -> (continuation of Deferred@1925896822 after retry RPC after error@1614981742) -> (continuation of Deferred@331085019 after retry RPC after error@2027145963) -> (continuation of Deferred@1550759661 after retry RPC after error@696936148) -> (continuation of Deferred@1708872210 after retry RPC after error@62474448) -> (continuation of Deferred@717840276 after retry RPC@1343168735), errback=(continuation of Deferred@107943165 after retry RPC after error@1753177269) -> (continuation of Deferred@948527221 after retry RPC after error@217903080) -> (continuation of Deferred@1925896822 after retry RPC after error@1614981742) -> (continuation of Deferred@331085019 after retry RPC after error@2027145963) -> (continuation of Deferred@1550759661 after retry RPC after error@696936148) -> (continuation of Deferred@1708872210 after retry RPC after error@62474448) -> (continuation of Deferred@717840276 after retry RPC@1343168735)))
15:14:15.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.613678 18581 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:37648:
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
15:14:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
15:14:15.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:15.665088 18581 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
15:14:15.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.790486 18958 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.137s from last keepalive heartbeat (effective timeout is 1.000s)
15:14:15.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:15.807565 18965 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.18.5.60:39939)
15:14:15.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:15.827752 19489 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
15:14:15.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.912745 19073 heartbeater.cc:344] Connected to a master server at 127.18.5.60:39939
15:14:15.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.915620 19395 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2647dd1730154da29c37a99de6575287" instance_seqno: 1765638847608787) as {username='slave'} at 127.18.5.2:34859; Asking this server to re-register.
15:14:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.916852 19073 heartbeater.cc:461] Registering TS with master...
15:14:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:15.919616 19395 ts_manager.cc:194] Registered new tserver with Master: 2647dd1730154da29c37a99de6575287 (127.18.5.2:44243)
15:14:15.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:15.940706 18568 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 -> Peer 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027): Couldn't send request to peer 779e0691761a4e4cb212855ae050a8a0. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.62:39027: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
15:14:16.239 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 2647dd1730154da29c37a99de6575287 (127.18.5.2:44243): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1765638850 last_transition_timestamp: 1765638855
15:14:16.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:16.240535 18568 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.5.62:39027: connect: Connection refused (error 111) [suppressed 10 similar messages]
15:14:16.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:16.443038 18568 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 -> Peer 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027): Couldn't send request to peer 779e0691761a4e4cb212855ae050a8a0. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.62:39027: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
15:14:16.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:16.912174 19074 heartbeater.cc:499] Master 127.18.5.61:34823 was elected leader, sending a full tablet report...
15:14:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:16.949388 18568 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9c577a162257444681464c098a4bc938 -> Peer 779e0691761a4e4cb212855ae050a8a0 (127.18.5.62:39027): Couldn't send request to peer 779e0691761a4e4cb212855ae050a8a0. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.62:39027: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
15:14:17.078 [DEBUG - main] (Connection.java:575) [peer master-127.18.5.62:39027(127.18.5.62:39027)] disconnecting while in state TERMINATED
15:14:17.078 [DEBUG - main] (Connection.java:575) [peer master-127.18.5.61:34823(127.18.5.61:34823)] disconnecting while in state READY
15:14:17.078 [DEBUG - main] (Connection.java:575) [peer e90a4dbb418e4689b953f8774bcd00a7(127.18.5.3:39467)] disconnecting while in state READY
15:14:17.078 [DEBUG - main] (Connection.java:575) [peer master-127.18.5.60:39939(127.18.5.60:39939)] disconnecting while in state READY
15:14:17.079 [DEBUG - kudu-nio-1] (Connection.java:239) [peer e90a4dbb418e4689b953f8774bcd00a7(127.18.5.3:39467)] handling channelInactive
15:14:17.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer e90a4dbb418e4689b953f8774bcd00a7(127.18.5.3:39467)] cleaning up while in state READY due to: connection closed
15:14:17.079 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:14:17.080 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.5.61:34823(127.18.5.61:34823)] handling channelInactive
15:14:17.080 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.61:34823(127.18.5.61:34823)] cleaning up while in state READY due to: connection closed
15:14:17.080 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.5.60:39939(127.18.5.60:39939)] handling channelInactive
15:14:17.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.5.60:39939(127.18.5.60:39939)] cleaning up while in state READY due to: connection closed
15:14:17.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:17.082039 18452 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu with pid 18763
15:14:17.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:17.121078 19101 connection.cc:537] client connection to 127.18.5.1:38599 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:14:17.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:17.121533 19101 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:14:17.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:17.126937 18452 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu with pid 18930
15:14:17.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:14:17.135505 19101 consensus_peers.cc:597] T 68e1d70dacc846b2adf1641511b487a2 P e90a4dbb418e4689b953f8774bcd00a7 -> Peer 6ddfc1734f9247beba1bc0f97ca83336 (127.18.5.1:38599): Couldn't send request to peer 6ddfc1734f9247beba1bc0f97ca83336. Status: Network error: Client connection negotiation failed: client connection to 127.18.5.1:38599: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:14:17.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:17.167147 18452 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu with pid 19078
15:14:17.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:17.210320 18452 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu with pid 18544
15:14:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:14:17.244748 18452 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskG_DMvO/build/tsan/bin/kudu with pid 19356
15:14:18.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:14:18Z chronyd exiting
15:14:19.419 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)