09:13:23.671 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
09:13:23.671 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0xFmrh/test-tmp
09:13:23.671 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:13:23.671 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0xFmrh/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:13:23.673 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
09:13:23.989 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:13:23.989 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:13:23.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:23.992908 27413 mini_oidc.cc:150] Starting JWKS server
09:13:23.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:23.996191 27413 webserver.cc:492] Webserver started at http://127.0.0.1:34909/ using document root <none> and password file <none>
09:13:23.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:23.996614 27413 mini_oidc.cc:186] Starting OIDC Discovery server
09:13:23.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:23.998291 27413 webserver.cc:492] Webserver started at http://127.0.0.1:33985/ using document root <none> and password file <none>
09:13:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T09:13:24Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:13:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T09:13:24Z Disabled control of system clock
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:24.059108 27413 external_mini_cluster.cc:1366] Running /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/logs
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data/info.pb
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.126:42303
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.126
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34909/jwks.json
09:13:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:13:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:13:24.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.356506 27425 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.357081 27425 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.357592 27425 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.389684 27425 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:13:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.389993 27425 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:13:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.390162 27425 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.390362 27425 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:13:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.390554 27425 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:13:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.390784 27425 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:13:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.390939 27425 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34909/jwks.json
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:24.427255 27425 master_runner.cc:386] Master server non-default flags:
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.126:42303
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34909/jwks.json
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data/info.pb
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.126
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/logs
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 08:43:20 UTC on 5fd53c4cbb9d
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9493
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:24.428638 27425 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:24.430312 27425 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:24.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.441426 27431 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:25.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:24.441743 27432 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:25.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:25.546284 27425 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.105s user 0.329s sys 0.771s
09:13:25.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:25.546739 27425 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.105s user 0.329s sys 0.771s
09:13:25.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:25.548317 27433 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1100 milliseconds
09:13:25.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:25.549758 27434 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:25.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.549835 27425 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:25.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.551141 27425 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:25.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.553786 27425 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.555146 27425 hybrid_clock.cc:648] HybridClock initialized: now 1765617205555116 us; error 40 us; skew 500 ppm
09:13:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.556022 27425 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:25.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.563298 27425 webserver.cc:492] Webserver started at http://127.26.197.126:37079/ using document root <none> and password file <none>
09:13:25.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.564239 27425 fs_manager.cc:362] Metadata directory not provided
09:13:25.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.564467 27425 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:25.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.564931 27425 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:25.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.569280 27425 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data/instance:
09:13:25.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a70b898276748bba9d75a67edd84c5d"
09:13:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:25 on dist-test-slave-k5d4"
09:13:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.570493 27425 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal/instance:
09:13:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a70b898276748bba9d75a67edd84c5d"
09:13:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:25 on dist-test-slave-k5d4"
09:13:25.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.578424 27425 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
09:13:25.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.584537 27441 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:25.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.585740 27425 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
09:13:25.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.586083 27425 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data,/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal
09:13:25.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a70b898276748bba9d75a67edd84c5d"
09:13:25.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:25 on dist-test-slave-k5d4"
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.586437 27425 fs_report.cc:389] FS layout report
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data/data
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.656773 27425 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:25.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.658304 27425 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.658797 27425 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:25.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.736927 27425 rpc_server.cc:307] RPC server started. Bound to: 127.26.197.126:42303
09:13:25.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.736999 27502 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.197.126:42303 every 8 connection(s)
09:13:25.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.739627 27425 server_base.cc:1179] Dumped server information to /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data/info.pb
09:13:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.740890 27413 external_mini_cluster.cc:1428] Started /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu as pid 27425
09:13:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.741328 27413 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal/instance
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.745293 27413 external_mini_cluster.cc:1366] Running /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/wal
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/data
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/logs
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/data/info.pb
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.125:37287
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.125
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34909/jwks.json
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:13:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.746031 27503 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.756476 27503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } has no permanent_uuid. Determining permanent_uuid...
09:13:25.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.783231 27503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } has no permanent_uuid. Determining permanent_uuid...
09:13:25.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:25.785509 27443 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:25.789485 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.125:37287: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.839044 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } attempt: 1
09:13:25.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:25.843106 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.125:37287: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:25.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:25.911594 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } attempt: 2
09:13:25.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:25.915760 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.125:37287: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:26.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:26.007189 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } attempt: 3
09:13:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.011456 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.125:37287: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.044905 27505 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.045485 27505 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:26.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.045986 27505 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.077951 27505 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:13:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.078253 27505 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:13:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.078418 27505 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.078610 27505 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:13:26.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.078821 27505 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:13:26.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.079011 27505 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:13:26.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.079154 27505 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34909/jwks.json
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:26.115592 27505 master_runner.cc:386] Master server non-default flags:
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/data
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/wal
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.125:37287
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34909/jwks.json
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/data/info.pb
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.125
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/logs
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 08:43:20 UTC on 5fd53c4cbb9d
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9493
09:13:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:26.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:26.116865 27505 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:26.118443 27505 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:26.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.129410 27514 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:26.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:26.154877 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } attempt: 4
09:13:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.160111 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.125:37287: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:26.459615 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } attempt: 5
09:13:26.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.464651 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.125:37287: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:27.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.012180 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } attempt: 6
09:13:27.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.021921 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.125:37287: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:27.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:26.129734 27515 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.255683 27517 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:27.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.258049 27516 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
09:13:27.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.258180 27505 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.129s user 0.464s sys 0.665s
09:13:27.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.258559 27505 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.130s user 0.464s sys 0.665s
09:13:27.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.258864 27505 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:27.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.260357 27505 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:27.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.263505 27505 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:27.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.265113 27505 hybrid_clock.cc:648] HybridClock initialized: now 1765617207265064 us; error 45 us; skew 500 ppm
09:13:27.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.266289 27505 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:27.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.275019 27505 webserver.cc:492] Webserver started at http://127.26.197.125:43339/ using document root <none> and password file <none>
09:13:27.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.276322 27505 fs_manager.cc:362] Metadata directory not provided
09:13:27.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.276611 27505 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:27.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.277179 27505 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.283504 27505 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/data/instance:
09:13:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80b3786ba7c94e0d96350abc26a3bee6"
09:13:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:27 on dist-test-slave-k5d4"
09:13:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.284986 27505 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/wal/instance:
09:13:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80b3786ba7c94e0d96350abc26a3bee6"
09:13:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:27 on dist-test-slave-k5d4"
09:13:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.294461 27505 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.001s sys 0.009s
09:13:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.302017 27526 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.303408 27505 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
09:13:27.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.303817 27505 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/data,/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/wal
09:13:27.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80b3786ba7c94e0d96350abc26a3bee6"
09:13:27.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:27 on dist-test-slave-k5d4"
09:13:27.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.304288 27505 fs_report.cc:389] FS layout report
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/wal
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/wal
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/data/data
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:27.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.405505 27505 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.407004 27505 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.407419 27505 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:27.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.486680 27505 rpc_server.cc:307] RPC server started. Bound to: 127.26.197.125:37287
09:13:27.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.486768 27587 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.197.125:37287 every 8 connection(s)
09:13:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.489430 27505 server_base.cc:1179] Dumped server information to /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/data/info.pb
09:13:27.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.495036 27588 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.499370 27413 external_mini_cluster.cc:1428] Started /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu as pid 27505
09:13:27.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.499853 27413 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-1/wal/instance
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.503115 27413 external_mini_cluster.cc:1366] Running /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/logs
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data/info.pb
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.124:35671
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.124
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34909/jwks.json
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:13:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:13:27.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.509101 27588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } has no permanent_uuid. Determining permanent_uuid...
09:13:27.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.529860 27588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } has no permanent_uuid. Determining permanent_uuid...
09:13:27.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.544193 27588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } has no permanent_uuid. Determining permanent_uuid...
09:13:27.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.546904 27529 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.550943 27588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:27.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.600509 27588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 1
09:13:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.604329 27588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.672811 27588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 2
09:13:27.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.676806 27588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.768253 27588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 3
09:13:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.772624 27588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:27.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.802866 27590 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:27.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.803424 27590 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:27.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.803901 27590 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:27.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.834504 27590 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:13:27.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.834841 27590 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:13:27.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.835088 27590 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:27.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.835299 27590 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:13:27.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.835486 27590 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:13:27.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.835673 27590 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:13:27.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.835814 27590 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34909/jwks.json
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.870666 27590 master_runner.cc:386] Master server non-default flags:
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.124:35671
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34909/jwks.json
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data/info.pb
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.124
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/logs
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 08:43:20 UTC on 5fd53c4cbb9d
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9493
09:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.871989 27590 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:27.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.873529 27590 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:27.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.884423 27600 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:27.916023 27588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 4
09:13:27.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.921794 27588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:28.084139 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } attempt: 7
09:13:28.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:28.114979 27503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } has no permanent_uuid. Determining permanent_uuid...
09:13:28.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:28.125288 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:28.183837 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 1
09:13:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:28.199427 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:28.221316 27588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 5
09:13:28.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:28.228595 27588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:28.280994 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 2
09:13:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:28.287587 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:28.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:28.373119 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 3
09:13:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:28.379854 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:28.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:28.520368 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 4
09:13:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:28.526989 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:28.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:28.776140 27588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 6
09:13:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:28.804651 27588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:28.810534 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 5
09:13:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:28.816654 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:29.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.286981 27599 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 27590
09:13:29.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.301793 27590 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.418s user 0.565s sys 0.848s
09:13:29.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.302436 27590 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.419s user 0.565s sys 0.848s
09:13:29.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:27.884781 27601 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.303738 27602 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1418 milliseconds
09:13:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.304805 27590 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.304847 27603 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.307837 27590 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:29.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.310246 27590 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.311587 27590 hybrid_clock.cc:648] HybridClock initialized: now 1765617209311542 us; error 38 us; skew 500 ppm
09:13:29.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.312423 27590 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:29.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.318634 27590 webserver.cc:492] Webserver started at http://127.26.197.124:35399/ using document root <none> and password file <none>
09:13:29.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.319659 27590 fs_manager.cc:362] Metadata directory not provided
09:13:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.319886 27590 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.320385 27590 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:29.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.324777 27590 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data/instance:
09:13:29.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba"
09:13:29.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:29 on dist-test-slave-k5d4"
09:13:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.325899 27590 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal/instance:
09:13:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba"
09:13:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:29 on dist-test-slave-k5d4"
09:13:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.333536 27590 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
09:13:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.340219 27616 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.341284 27590 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.341668 27590 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data,/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba"
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:29 on dist-test-slave-k5d4"
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.342001 27590 fs_report.cc:389] FS layout report
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data/data
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.369125 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 6
09:13:29.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.375842 27503 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.197.124:35671: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:29.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.400489 27590 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.401960 27590 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.402400 27590 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:29.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.484406 27590 rpc_server.cc:307] RPC server started. Bound to: 127.26.197.124:35671
09:13:29.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.484532 27679 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.197.124:35671 every 8 connection(s)
09:13:29.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.487296 27590 server_base.cc:1179] Dumped server information to /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data/info.pb
09:13:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.493341 27413 external_mini_cluster.cc:1428] Started /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu as pid 27590
09:13:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.493839 27413 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal/instance
09:13:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.494489 27680 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:29.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.509516 27680 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } has no permanent_uuid. Determining permanent_uuid...
09:13:29.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.531770 27680 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } has no permanent_uuid. Determining permanent_uuid...
09:13:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.541473 27680 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } has no permanent_uuid. Determining permanent_uuid...
09:13:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.556764 27680 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: Bootstrap starting.
09:13:29.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.563007 27680 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: Neither blocks nor log segments found. Creating new log.
09:13:29.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.564648 27680 log.cc:826] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: Log is configured to *not* fsync() on all Append() calls
09:13:29.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.569859 27680 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: No bootstrap required, opened a new log
09:13:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.588660 27680 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.589493 27680 raft_consensus.cc:385] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.589708 27680 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1cd9f73088cd4a8fa0cdc72a8dd908ba, State: Initialized, Role: FOLLOWER
09:13:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.590373 27680 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:29.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.592530 27689 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:29.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.593127 27689 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: This master's current role is: FOLLOWER
09:13:29.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.593995 27680 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: configured and running, proceeding with master startup.
09:13:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.602728 27689 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:13:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.603310 27689 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:29.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.605876 27689 leader_election.cc:290] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303), 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287)
09:13:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.622924 27700 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:13:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.623274 27700 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:13:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.617928 27590 master_runner.cc:186] Error getting master registration for 127.26.197.126:42303: OK, instance_id { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" instance_seqno: 1765617205719614 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:13:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.649183 27590 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:13:29.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.657529 27624 tablet.cc:2378] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:13:29.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.865120 27588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 7
09:13:29.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.886299 27588 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6: Bootstrap starting.
09:13:29.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.893721 27588 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6: Neither blocks nor log segments found. Creating new log.
09:13:29.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.896185 27588 log.cc:826] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6: Log is configured to *not* fsync() on all Append() calls
09:13:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.903512 27588 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6: No bootstrap required, opened a new log
09:13:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.922312 27588 raft_consensus.cc:359] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.923060 27588 raft_consensus.cc:385] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.923343 27588 raft_consensus.cc:740] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 80b3786ba7c94e0d96350abc26a3bee6, State: Initialized, Role: FOLLOWER
09:13:29.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.924150 27588 consensus_queue.cc:260] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.926667 27704 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.927667 27704 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: This master's current role is: FOLLOWER
09:13:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.928767 27588 sys_catalog.cc:565] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: configured and running, proceeding with master startup.
09:13:29.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.929054 27553 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "80b3786ba7c94e0d96350abc26a3bee6" is_pre_election: true
09:13:29.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.933096 27553 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1cd9f73088cd4a8fa0cdc72a8dd908ba in term 0.
09:13:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.934790 27553 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:47052 (request call id 1) took 327 ms (client timeout 381 ms). Trace:
09:13:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.935338 27553 rpcz_store.cc:269] 1213 09:13:29.607257 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:13:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:29.607379 (+ 122us) service_pool.cc:225] Handling call
09:13:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:29.934688 (+327309us) inbound_call.cc:173] Queueing success response
09:13:29.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:13:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.937453 27620 leader_election.cc:304] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1cd9f73088cd4a8fa0cdc72a8dd908ba, 80b3786ba7c94e0d96350abc26a3bee6; no voters:
09:13:29.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.938476 27689 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 0 FOLLOWER]: Leader pre-election won for term 1
09:13:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.938849 27689 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:13:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.939178 27689 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 0 FOLLOWER]: Advancing to term 1
09:13:29.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.946084 27689 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:29.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.950450 27553 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "80b3786ba7c94e0d96350abc26a3bee6"
09:13:29.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.951046 27553 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 0 FOLLOWER]: Advancing to term 1
09:13:29.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.957916 27553 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1cd9f73088cd4a8fa0cdc72a8dd908ba in term 1.
09:13:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.959277 27620 leader_election.cc:304] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1cd9f73088cd4a8fa0cdc72a8dd908ba, 80b3786ba7c94e0d96350abc26a3bee6; no voters:
09:13:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.971819 27689 leader_election.cc:290] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 1 election: Requested vote from peers 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303), 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287)
09:13:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.972447 27689 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 1 FOLLOWER]: Leader election won for term 1
09:13:29.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.972882 27689 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 1 LEADER]: Becoming Leader. State: Replica: 1cd9f73088cd4a8fa0cdc72a8dd908ba, State: Running, Role: LEADER
09:13:29.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.973773 27689 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:29.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.987680 27622 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.26.197.126:42303 timed out after 0.381s (SENT)
09:13:29.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.990108 27716 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:13:29.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:29.990469 27716 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:13:30.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.010835 27715 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1cd9f73088cd4a8fa0cdc72a8dd908ba. Latest consensus state: current_term: 1 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.011900 27715 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: This master's current role is: LEADER
09:13:30.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.026525 27720 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:13:30.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:29.971923 27505 master_runner.cc:186] Error getting master registration for 127.26.197.126:42303: OK, instance_id { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" instance_seqno: 1765617205719614 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:13:30.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.039692 27505 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:13:30.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.044102 27720 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:13:30.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.065548 27553 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 1 FOLLOWER]: Refusing update from remote peer 1cd9f73088cd4a8fa0cdc72a8dd908ba: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:13:30.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.067502 27715 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [LEADER]: Connected to new peer: Peer: permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:13:30.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.107210 27704 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1cd9f73088cd4a8fa0cdc72a8dd908ba. Latest consensus state: current_term: 1 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:30.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.107918 27704 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: This master's current role is: FOLLOWER
09:13:30.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.120177 27715 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.122210 27715 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: This master's current role is: LEADER
09:13:30.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.133144 27704 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.133855 27704 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: This master's current role is: FOLLOWER
09:13:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.141126 27720 catalog_manager.cc:1357] Generated new cluster ID: 812ace05a5054ff79e14cf188ae3b8f5
09:13:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.141453 27720 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:13:30.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.184958 27720 catalog_manager.cc:1380] Generated new certificate authority record
09:13:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.187748 27720 catalog_manager.cc:1514] Loading token signing keys...
09:13:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.222966 27720 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: Generated new TSK 0
09:13:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.224094 27720 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:13:30.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.260326 27413 external_mini_cluster.cc:1366] Running /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:30.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:30.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/wal
09:13:30.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/data
09:13:30.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/logs
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/data/info.pb
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.65:0
09:13:30.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.197.65
09:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.65
09:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:13:30.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.384522 27622 leader_election.cc:336] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Timed out: RequestConsensusVote RPC to 127.26.197.126:42303 timed out after 0.381s (SENT)
09:13:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.409320 27503 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } attempt: 7
09:13:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.477860 27503 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: Bootstrap starting.
09:13:30.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.497498 27503 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: Neither blocks nor log segments found. Creating new log.
09:13:30.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.499825 27503 log.cc:826] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: Log is configured to *not* fsync() on all Append() calls
09:13:30.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.508615 27503 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: No bootstrap required, opened a new log
09:13:30.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.560415 27503 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.570931 27503 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.571533 27503 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a70b898276748bba9d75a67edd84c5d, State: Initialized, Role: FOLLOWER
09:13:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.572713 27503 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:30.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.584245 27731 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:30.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.585263 27731 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: This master's current role is: FOLLOWER
09:13:30.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.585983 27503 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: configured and running, proceeding with master startup.
09:13:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.586158 27467 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3a70b898276748bba9d75a67edd84c5d"
09:13:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.586930 27467 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 0 FOLLOWER]: Advancing to term 1
09:13:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.590962 27464 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3a70b898276748bba9d75a67edd84c5d"
09:13:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.592000 27464 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:59910 (request call id 6) took 283 ms (client timeout 358 ms). Trace:
09:13:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.592365 27464 rpcz_store.cc:269] 1213 09:13:30.308460 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:13:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:30.308608 (+ 148us) service_pool.cc:225] Handling call
09:13:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:30.591918 (+283310us) inbound_call.cc:173] Queueing success response
09:13:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:13:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.594501 27468 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3a70b898276748bba9d75a67edd84c5d" is_pre_election: true
09:13:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.595448 27468 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:59910 (request call id 1) took 989 ms (client timeout 381 ms). Trace:
09:13:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.595824 27468 rpcz_store.cc:269] 1213 09:13:29.606560 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:13:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:29.606686 (+ 126us) service_pool.cc:225] Handling call
09:13:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:30.595372 (+988686us) inbound_call.cc:173] Queueing success response
09:13:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:13:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.590523 27466 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3a70b898276748bba9d75a67edd84c5d" is_pre_election: true
09:13:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.599740 27466 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:59910 (request call id 4) took 596 ms (client timeout 381 ms). Trace:
09:13:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.600077 27466 rpcz_store.cc:269] 1213 09:13:30.002938 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:13:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:30.003081 (+ 143us) service_pool.cc:225] Handling call
09:13:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:30.599669 (+596588us) inbound_call.cc:173] Queueing success response
09:13:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:13:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.609308 27467 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1cd9f73088cd4a8fa0cdc72a8dd908ba in term 1.
09:13:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.610699 27465 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 1 FOLLOWER]: Refusing update from remote peer 1cd9f73088cd4a8fa0cdc72a8dd908ba: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:13:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.615491 27725 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [LEADER]: Connected to new peer: Peer: permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
09:13:30.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.626961 27467 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:59910 (request call id 3) took 678 ms (client timeout 358 ms). Trace:
09:13:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.627683 27467 rpcz_store.cc:269] 1213 09:13:29.948521 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:13:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:29.948673 (+ 152us) service_pool.cc:225] Handling call
09:13:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:30.626885 (+678212us) inbound_call.cc:173] Queueing success response
09:13:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:13:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.677372 27731 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1cd9f73088cd4a8fa0cdc72a8dd908ba. Latest consensus state: current_term: 1 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.677441 27741 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.678194 27731 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: This master's current role is: FOLLOWER
09:13:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.678476 27741 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: This master's current role is: FOLLOWER
09:13:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.684443 27747 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:13:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.720467 27689 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.721164 27689 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: This master's current role is: LEADER
09:13:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.889588 27727 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.890230 27727 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.891269 27727 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:30.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.947999 27727 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.948961 27727 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:13:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.949276 27727 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:13:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.949533 27727 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:13:30.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:30.950261 27727 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.197.65
09:13:30.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.996170 27716 catalog_manager.cc:1269] Loaded cluster ID: 812ace05a5054ff79e14cf188ae3b8f5
09:13:30.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:30.996675 27716 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6: loading cluster ID for follower catalog manager: success
09:13:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:31.004169 27716 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6: acquiring CA information for follower catalog manager: success
09:13:31.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:31.009932 27716 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:31.012849 27727 tablet_server_runner.cc:78] Tablet server non-default flags:
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/data
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/wal
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.65:0
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/data/info.pb
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.65
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.197.65
09:13:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/logs
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 08:43:20 UTC on 5fd53c4cbb9d
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9493
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:31.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:31.014441 27727 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:31.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:31.016582 27727 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:31.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:31.032651 27755 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:31.704651 27747 catalog_manager.cc:1269] Loaded cluster ID: 812ace05a5054ff79e14cf188ae3b8f5
09:13:31.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:31.704931 27747 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: loading cluster ID for follower catalog manager: success
09:13:31.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:31.722270 27747 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: acquiring CA information for follower catalog manager: success
09:13:31.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:31.754513 27747 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:13:32.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:31.040624 27727 server_base.cc:1047] running on GCE node
09:13:32.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:31.041903 27756 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:32.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:31.042091 27758 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:32.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.387702 27727 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.396550 27727 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:32.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.398208 27727 hybrid_clock.cc:648] HybridClock initialized: now 1765617212398140 us; error 78 us; skew 500 ppm
09:13:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.399508 27727 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:32.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.424234 27727 webserver.cc:492] Webserver started at http://127.26.197.65:41455/ using document root <none> and password file <none>
09:13:32.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.425791 27727 fs_manager.cc:362] Metadata directory not provided
09:13:32.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.426246 27727 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.426944 27727 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:32.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.433367 27727 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/data/instance:
09:13:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b03cd02817ea44cfa620f4c79f743171"
09:13:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:32 on dist-test-slave-k5d4"
09:13:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.435158 27727 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/wal/instance:
09:13:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b03cd02817ea44cfa620f4c79f743171"
09:13:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:32 on dist-test-slave-k5d4"
09:13:32.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.450551 27727 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.009s sys 0.004s
09:13:32.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.462301 27767 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:32.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.464359 27727 fs_manager.cc:730] Time spent opening block manager: real 0.009s user 0.003s sys 0.004s
09:13:32.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.464964 27727 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/data,/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/wal
09:13:32.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b03cd02817ea44cfa620f4c79f743171"
09:13:32.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:32 on dist-test-slave-k5d4"
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.465600 27727 fs_report.cc:389] FS layout report
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/wal
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/wal
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/data/data
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:32.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.567437 27727 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:32.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.569393 27727 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:32.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.569934 27727 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:32.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.583115 27727 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:13:32.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.583364 27727 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:13:32.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.583674 27727 ts_tablet_manager.cc:616] Registered 0 tablets
09:13:32.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.583873 27727 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:13:32.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.820399 27727 rpc_server.cc:307] RPC server started. Bound to: 127.26.197.65:45603
09:13:32.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.820902 27891 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.197.65:45603 every 8 connection(s)
09:13:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.823146 27727 server_base.cc:1179] Dumped server information to /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/data/info.pb
09:13:32.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.829584 27413 external_mini_cluster.cc:1428] Started /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu as pid 27727
09:13:32.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.830067 27413 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-0/wal/instance
09:13:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.832376 27894 heartbeater.cc:344] Connected to a master server at 127.26.197.126:42303
09:13:32.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.838541 27894 heartbeater.cc:461] Registering TS with master...
09:13:32.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.839973 27894 heartbeater.cc:507] Master 127.26.197.126:42303 requested a full tablet report, sending...
09:13:32.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.843667 27458 ts_manager.cc:194] Registered new tserver with Master: b03cd02817ea44cfa620f4c79f743171 (127.26.197.65:45603)
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.846280 27413 external_mini_cluster.cc:1366] Running /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/wal
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/data
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/logs
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/data/info.pb
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.66:0
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.197.66
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.66
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:13:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:13:32.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.853641 27892 heartbeater.cc:344] Connected to a master server at 127.26.197.125:37287
09:13:32.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.854008 27892 heartbeater.cc:461] Registering TS with master...
09:13:32.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.854909 27892 heartbeater.cc:507] Master 127.26.197.125:37287 requested a full tablet report, sending...
09:13:32.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.857304 27893 heartbeater.cc:344] Connected to a master server at 127.26.197.124:35671
09:13:32.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.857649 27893 heartbeater.cc:461] Registering TS with master...
09:13:32.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.858000 27543 ts_manager.cc:194] Registered new tserver with Master: b03cd02817ea44cfa620f4c79f743171 (127.26.197.65:45603)
09:13:32.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.858498 27893 heartbeater.cc:507] Master 127.26.197.124:35671 requested a full tablet report, sending...
09:13:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.861014 27635 ts_manager.cc:194] Registered new tserver with Master: b03cd02817ea44cfa620f4c79f743171 (127.26.197.65:45603)
09:13:32.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:32.863804 27635 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.197.65:36415
09:13:33.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:33.163081 27897 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:33.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:33.163551 27897 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:33.164106 27897 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:33.197846 27897 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:33.198427 27897 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:13:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:33.198608 27897 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:13:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:33.198791 27897 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:13:33.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:33.199214 27897 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.197.66
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:33.235646 27897 tablet_server_runner.cc:78] Tablet server non-default flags:
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/data
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/wal
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.66:0
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/data/info.pb
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.66
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.197.66
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/logs
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 08:43:20 UTC on 5fd53c4cbb9d
09:13:33.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9493
09:13:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:33.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:33.236901 27897 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:33.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:33.238463 27897 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:33.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:33.251822 27903 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:33.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:33.867658 27893 heartbeater.cc:499] Master 127.26.197.124:35671 was elected leader, sending a full tablet report...
09:13:34.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:34.653959 27902 debug-util.cc:398] Leaking SignalData structure 0x7b0800037660 after lost signal to thread 27897
09:13:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:34.792531 27902 kernel_stack_watchdog.cc:198] Thread 27897 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
09:13:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:13:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:13:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:13:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:13:34.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:33.253789 27904 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:34.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:34.794155 27897 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.542s user 0.589s sys 0.825s
09:13:34.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:34.794492 27897 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.542s user 0.589s sys 0.825s
09:13:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:34.798828 27905 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1545 milliseconds
09:13:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:34.799645 27906 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.799714 27897 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:34.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.800983 27897 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.807361 27897 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:34.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.808883 27897 hybrid_clock.cc:648] HybridClock initialized: now 1765617214808770 us; error 121 us; skew 500 ppm
09:13:34.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.809835 27897 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.816077 27897 webserver.cc:492] Webserver started at http://127.26.197.66:38949/ using document root <none> and password file <none>
09:13:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.817015 27897 fs_manager.cc:362] Metadata directory not provided
09:13:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.817239 27897 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.817742 27897 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:34.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.821961 27897 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/data/instance:
09:13:34.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a968a863d78b44118abd134c71c37377"
09:13:34.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:34 on dist-test-slave-k5d4"
09:13:34.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.823127 27897 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/wal/instance:
09:13:34.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a968a863d78b44118abd134c71c37377"
09:13:34.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:34 on dist-test-slave-k5d4"
09:13:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.830964 27897 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
09:13:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.837149 27916 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:34.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.838490 27897 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.838821 27897 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/data,/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/wal
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a968a863d78b44118abd134c71c37377"
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:34 on dist-test-slave-k5d4"
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.839131 27897 fs_report.cc:389] FS layout report
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/wal
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/wal
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/data/data
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:34.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.911084 27897 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:34.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.912813 27897 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:34.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.913372 27897 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:34.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.923558 27897 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:13:34.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.923863 27897 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:13:34.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.924187 27897 ts_tablet_manager.cc:616] Registered 0 tablets
09:13:34.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:34.924417 27897 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:13:35.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.092083 27897 rpc_server.cc:307] RPC server started. Bound to: 127.26.197.66:43885
09:13:35.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.092581 28040 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.197.66:43885 every 8 connection(s)
09:13:35.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.094828 27897 server_base.cc:1179] Dumped server information to /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/data/info.pb
09:13:35.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.101169 27413 external_mini_cluster.cc:1428] Started /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu as pid 27897
09:13:35.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.101823 27413 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-1/wal/instance
09:13:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.107920 28043 heartbeater.cc:344] Connected to a master server at 127.26.197.126:42303
09:13:35.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.108740 28043 heartbeater.cc:461] Registering TS with master...
09:13:35.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.110646 28043 heartbeater.cc:507] Master 127.26.197.126:42303 requested a full tablet report, sending...
09:13:35.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.111150 28041 heartbeater.cc:344] Connected to a master server at 127.26.197.125:37287
09:13:35.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.111523 28041 heartbeater.cc:461] Registering TS with master...
09:13:35.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.112401 28041 heartbeater.cc:507] Master 127.26.197.125:37287 requested a full tablet report, sending...
09:13:35.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.114022 27457 ts_manager.cc:194] Registered new tserver with Master: a968a863d78b44118abd134c71c37377 (127.26.197.66:43885)
09:13:35.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.115082 27543 ts_manager.cc:194] Registered new tserver with Master: a968a863d78b44118abd134c71c37377 (127.26.197.66:43885)
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.120682 27413 external_mini_cluster.cc:1366] Running /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/wal
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/data
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/logs
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/data/info.pb
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.67:0
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.197.67
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.67
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.120990 28042 heartbeater.cc:344] Connected to a master server at 127.26.197.124:35671
09:13:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.121392 28042 heartbeater.cc:461] Registering TS with master...
09:13:35.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.122259 28042 heartbeater.cc:507] Master 127.26.197.124:35671 requested a full tablet report, sending...
09:13:35.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.124217 27635 ts_manager.cc:194] Registered new tserver with Master: a968a863d78b44118abd134c71c37377 (127.26.197.66:43885)
09:13:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.125855 27635 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.197.66:50471
09:13:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:35.418603 28046 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:35.419184 28046 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:35.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:35.419852 28046 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:35.451807 28046 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:35.452468 28046 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:13:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:35.452705 28046 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:13:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:35.452932 28046 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:13:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:35.453410 28046 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.197.67
09:13:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.490372 28046 tablet_server_runner.cc:78] Tablet server non-default flags:
09:13:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/data
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/wal
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.67:0
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/data/info.pb
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.67
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.197.67
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/logs
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:13:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:13:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 08:43:20 UTC on 5fd53c4cbb9d
09:13:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9493
09:13:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.491979 28046 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:35.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:35.494083 28046 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:35.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:35.508989 28053 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:36.128753 28042 heartbeater.cc:499] Master 127.26.197.124:35671 was elected leader, sending a full tablet report...
09:13:37.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:36.910486 28051 debug-util.cc:398] Leaking SignalData structure 0x7b0800037660 after lost signal to thread 28046
09:13:37.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:37.055361 28046 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.543s user 0.580s sys 0.913s
09:13:37.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:37.055686 28046 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.543s user 0.580s sys 0.913s
09:13:37.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:35.512636 28052 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:37.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:37.057814 28055 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:37.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:37.060810 28054 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1542 milliseconds
09:13:37.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.060830 28046 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:37.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.062306 28046 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:37.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.064497 28046 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.065855 28046 hybrid_clock.cc:648] HybridClock initialized: now 1765617217065823 us; error 37 us; skew 500 ppm
09:13:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.066737 28046 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:37.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.073172 28046 webserver.cc:492] Webserver started at http://127.26.197.67:34301/ using document root <none> and password file <none>
09:13:37.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.074136 28046 fs_manager.cc:362] Metadata directory not provided
09:13:37.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.074383 28046 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:37.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.074903 28046 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:13:37.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.079169 28046 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/data/instance:
09:13:37.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8d35d68bd544dd1bb30304d9ec69d5c"
09:13:37.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:37 on dist-test-slave-k5d4"
09:13:37.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.080454 28046 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/wal/instance:
09:13:37.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8d35d68bd544dd1bb30304d9ec69d5c"
09:13:37.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:37 on dist-test-slave-k5d4"
09:13:37.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.087471 28046 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
09:13:37.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.093503 28064 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.094837 28046 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.001s
09:13:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.095199 28046 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/data,/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/wal
09:13:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8d35d68bd544dd1bb30304d9ec69d5c"
09:13:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:37 on dist-test-slave-k5d4"
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.095551 28046 fs_report.cc:389] FS layout report
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/wal
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/wal
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/data/data
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:37.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.160352 28046 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:37.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.161870 28046 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:37.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.162376 28046 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.172232 28046 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:13:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.172477 28046 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:13:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.172776 28046 ts_tablet_manager.cc:616] Registered 0 tablets
09:13:37.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.172946 28046 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:13:37.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.335369 28046 rpc_server.cc:307] RPC server started. Bound to: 127.26.197.67:45421
09:13:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.335865 28188 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.197.67:45421 every 8 connection(s)
09:13:37.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.337841 28046 server_base.cc:1179] Dumped server information to /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/data/info.pb
09:13:37.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.346333 27413 external_mini_cluster.cc:1428] Started /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu as pid 28046
09:13:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.347482 27413 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/ts-2/wal/instance
09:13:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.349841 28191 heartbeater.cc:344] Connected to a master server at 127.26.197.126:42303
09:13:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.350577 28191 heartbeater.cc:461] Registering TS with master...
09:13:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.357113 28189 heartbeater.cc:344] Connected to a master server at 127.26.197.125:37287
09:13:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.357542 28189 heartbeater.cc:461] Registering TS with master...
09:13:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.357725 28191 heartbeater.cc:507] Master 127.26.197.126:42303 requested a full tablet report, sending...
09:13:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.358685 28189 heartbeater.cc:507] Master 127.26.197.125:37287 requested a full tablet report, sending...
09:13:37.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.361755 27457 ts_manager.cc:194] Registered new tserver with Master: a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67:45421)
09:13:37.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.361709 27543 ts_manager.cc:194] Registered new tserver with Master: a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67:45421)
09:13:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.377887 28190 heartbeater.cc:344] Connected to a master server at 127.26.197.124:35671
09:13:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.378225 28190 heartbeater.cc:461] Registering TS with master...
09:13:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.379076 28190 heartbeater.cc:507] Master 127.26.197.124:35671 requested a full tablet report, sending...
09:13:37.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.381021 27635 ts_manager.cc:194] Registered new tserver with Master: a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67:45421)
09:13:37.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.382597 27635 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.197.67:59493
09:13:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.391114 27413 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
09:13:37.406 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:13:37.406 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
09:13:37.407 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.26.197.126"
port: 42303
}
bound_http_address {
host: "127.26.197.126"
port: 37079
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.26.197.125"
port: 37287
}
bound_http_address {
host: "127.26.197.125"
port: 43339
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.26.197.124"
port: 35671
}
bound_http_address {
host: "127.26.197.124"
port: 35399
}
}
}
09:13:37.408 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:13:37.408 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.26.197.65"
port: 45603
}
bound_http_address {
host: "127.26.197.65"
port: 41455
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.26.197.66"
port: 43885
}
bound_http_address {
host: "127.26.197.66"
port: 38949
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.26.197.67"
port: 45421
}
bound_http_address {
host: "127.26.197.67"
port: 34301
}
}
}
09:13:37.409 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
09:13:37.410 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
09:13:37.412 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:37.414 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.197.124:35671(127.26.197.124:35671)] connecting to peer
09:13:37.415 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.26.197.125:37287(127.26.197.125:37287)] Successfully connected to peer
09:13:37.418 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.197.126:42303(127.26.197.126:42303)] connecting to peer
09:13:37.419 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.26.197.124:35671(127.26.197.124:35671)] Successfully connected to peer
09:13:37.419 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.26.197.125
09:13:37.421 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.26.197.124
09:13:37.423 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.26.197.126:42303(127.26.197.126:42303)] Successfully connected to peer
09:13:37.432 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.26.197.126
09:13:37.438 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7be53790, L:/127.0.0.1:45252 - R:/127.26.197.124:35671] using SASL/PLAIN
09:13:37.441 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:13:37.442 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=30), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [29ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, deferred=null) after lookup
09:13:37.446 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf27176e9, L:/127.0.0.1:33554 - R:/127.26.197.125:37287] using SASL/PLAIN
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.445148 27635 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45252:
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:37.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:37.447 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb44db37e, L:/127.0.0.1:50144 - R:/127.26.197.126:42303] using SASL/PLAIN
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:13:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:37.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:37.448894 27635 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:13:37.481 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 03eaf326a2654e0dab58aa42e3c28b4a
09:13:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.559159 27975 tablet_service.cc:1505] Processing CreateTablet for tablet 8b5bb375b623431b90ededae66a9070a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:13:37.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.561203 27975 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8b5bb375b623431b90ededae66a9070a. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:37.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.559489 27976 tablet_service.cc:1505] Processing CreateTablet for tablet 51549a3ef4aa4d7597f571ac38fa192f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:13:37.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.563462 27976 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 51549a3ef4aa4d7597f571ac38fa192f. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:37.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.569023 28123 tablet_service.cc:1505] Processing CreateTablet for tablet 8b5bb375b623431b90ededae66a9070a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:13:37.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.573081 28123 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8b5bb375b623431b90ededae66a9070a. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:37.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.571466 28124 tablet_service.cc:1505] Processing CreateTablet for tablet 51549a3ef4aa4d7597f571ac38fa192f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:13:37.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.574738 28124 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 51549a3ef4aa4d7597f571ac38fa192f. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.581040 27827 tablet_service.cc:1505] Processing CreateTablet for tablet 51549a3ef4aa4d7597f571ac38fa192f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:13:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.583168 27827 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 51549a3ef4aa4d7597f571ac38fa192f. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.581015 27826 tablet_service.cc:1505] Processing CreateTablet for tablet 8b5bb375b623431b90ededae66a9070a (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:13:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.588655 27826 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8b5bb375b623431b90ededae66a9070a. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:37.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.624383 28215 tablet_bootstrap.cc:492] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377: Bootstrap starting.
09:13:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.632562 28215 tablet_bootstrap.cc:654] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377: Neither blocks nor log segments found. Creating new log.
09:13:37.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.635114 28215 log.cc:826] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377: Log is configured to *not* fsync() on all Append() calls
09:13:37.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.644376 28218 tablet_bootstrap.cc:492] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171: Bootstrap starting.
09:13:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.644843 28216 tablet_bootstrap.cc:492] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c: Bootstrap starting.
09:13:37.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.648738 28215 tablet_bootstrap.cc:492] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377: No bootstrap required, opened a new log
09:13:37.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.649328 28215 ts_tablet_manager.cc:1403] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377: Time spent bootstrapping tablet: real 0.026s user 0.018s sys 0.004s
09:13:37.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.650254 28218 tablet_bootstrap.cc:654] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171: Neither blocks nor log segments found. Creating new log.
09:13:37.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.652009 28216 tablet_bootstrap.cc:654] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c: Neither blocks nor log segments found. Creating new log.
09:13:37.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.652153 28218 log.cc:826] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171: Log is configured to *not* fsync() on all Append() calls
09:13:37.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.654301 28216 log.cc:826] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c: Log is configured to *not* fsync() on all Append() calls
09:13:37.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.658318 28218 tablet_bootstrap.cc:492] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171: No bootstrap required, opened a new log
09:13:37.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.658921 28218 ts_tablet_manager.cc:1403] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171: Time spent bootstrapping tablet: real 0.015s user 0.006s sys 0.008s
09:13:37.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.662434 28216 tablet_bootstrap.cc:492] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c: No bootstrap required, opened a new log
09:13:37.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.663026 28216 ts_tablet_manager.cc:1403] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c: Time spent bootstrapping tablet: real 0.019s user 0.018s sys 0.000s
09:13:37.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.677150 28215 raft_consensus.cc:359] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:37.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.678129 28215 raft_consensus.cc:385] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:37.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.678444 28215 raft_consensus.cc:740] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a968a863d78b44118abd134c71c37377, State: Initialized, Role: FOLLOWER
09:13:37.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.678114 28218 raft_consensus.cc:359] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } }
09:13:37.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.678860 28218 raft_consensus.cc:385] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:37.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.679214 28218 raft_consensus.cc:740] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b03cd02817ea44cfa620f4c79f743171, State: Initialized, Role: FOLLOWER
09:13:37.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.679381 28215 consensus_queue.cc:260] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:37.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.680001 28218 consensus_queue.cc:260] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } }
09:13:37.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.683308 28215 ts_tablet_manager.cc:1434] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377: Time spent starting tablet: real 0.034s user 0.028s sys 0.008s
09:13:37.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.688604 28215 tablet_bootstrap.cc:492] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377: Bootstrap starting.
09:13:37.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.691856 28216 raft_consensus.cc:359] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:37.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.692883 28216 raft_consensus.cc:385] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:37.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.693181 28216 raft_consensus.cc:740] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8d35d68bd544dd1bb30304d9ec69d5c, State: Initialized, Role: FOLLOWER
09:13:37.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.694196 28216 consensus_queue.cc:260] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:37.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.695127 28215 tablet_bootstrap.cc:654] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377: Neither blocks nor log segments found. Creating new log.
09:13:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.708388 28218 ts_tablet_manager.cc:1434] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171: Time spent starting tablet: real 0.049s user 0.025s sys 0.018s
09:13:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.708840 28216 ts_tablet_manager.cc:1434] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c: Time spent starting tablet: real 0.046s user 0.033s sys 0.012s
09:13:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.709563 28218 tablet_bootstrap.cc:492] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171: Bootstrap starting.
09:13:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.709507 28190 heartbeater.cc:499] Master 127.26.197.124:35671 was elected leader, sending a full tablet report...
09:13:37.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.711563 28216 tablet_bootstrap.cc:492] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c: Bootstrap starting.
09:13:37.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.717859 28216 tablet_bootstrap.cc:654] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c: Neither blocks nor log segments found. Creating new log.
09:13:37.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.719326 28215 tablet_bootstrap.cc:492] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377: No bootstrap required, opened a new log
09:13:37.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.719780 28215 ts_tablet_manager.cc:1403] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377: Time spent bootstrapping tablet: real 0.031s user 0.015s sys 0.001s
09:13:37.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.724205 28215 raft_consensus.cc:359] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } }
09:13:37.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.724982 28215 raft_consensus.cc:385] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:37.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.725272 28215 raft_consensus.cc:740] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a968a863d78b44118abd134c71c37377, State: Initialized, Role: FOLLOWER
09:13:37.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.726562 28218 tablet_bootstrap.cc:654] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171: Neither blocks nor log segments found. Creating new log.
09:13:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.726073 28215 consensus_queue.cc:260] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } }
09:13:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.738319 28218 tablet_bootstrap.cc:492] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171: No bootstrap required, opened a new log
09:13:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.738787 28218 ts_tablet_manager.cc:1403] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171: Time spent bootstrapping tablet: real 0.029s user 0.009s sys 0.007s
09:13:37.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.741382 28218 raft_consensus.cc:359] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:37.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.742048 28218 raft_consensus.cc:385] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:37.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.742319 28218 raft_consensus.cc:740] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b03cd02817ea44cfa620f4c79f743171, State: Initialized, Role: FOLLOWER
09:13:37.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.743197 28218 consensus_queue.cc:260] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:37.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.745225 28218 ts_tablet_manager.cc:1434] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171: Time spent starting tablet: real 0.006s user 0.003s sys 0.002s
09:13:37.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.759899 28215 ts_tablet_manager.cc:1434] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377: Time spent starting tablet: real 0.040s user 0.001s sys 0.005s
09:13:37.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.761318 28216 tablet_bootstrap.cc:492] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c: No bootstrap required, opened a new log
09:13:37.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.761762 28216 ts_tablet_manager.cc:1403] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c: Time spent bootstrapping tablet: real 0.050s user 0.023s sys 0.024s
09:13:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.764240 28216 raft_consensus.cc:359] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } }
09:13:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.764947 28216 raft_consensus.cc:385] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.765213 28216 raft_consensus.cc:740] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8d35d68bd544dd1bb30304d9ec69d5c, State: Initialized, Role: FOLLOWER
09:13:37.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.765874 28222 raft_consensus.cc:493] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:13:37.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.766607 28222 raft_consensus.cc:515] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:37.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.766949 28216 consensus_queue.cc:260] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } }
09:13:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.774194 28216 ts_tablet_manager.cc:1434] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c: Time spent starting tablet: real 0.012s user 0.007s sys 0.000s
09:13:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.782567 28222 leader_election.cc:290] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a968a863d78b44118abd134c71c37377 (127.26.197.66:43885), a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67:45421)
09:13:37.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.796592 27996 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "8b5bb375b623431b90ededae66a9070a" candidate_uuid: "b03cd02817ea44cfa620f4c79f743171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a968a863d78b44118abd134c71c37377" is_pre_election: true
09:13:37.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.797596 27996 raft_consensus.cc:2468] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b03cd02817ea44cfa620f4c79f743171 in term 0.
09:13:37.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.799603 27768 leader_election.cc:304] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a968a863d78b44118abd134c71c37377, b03cd02817ea44cfa620f4c79f743171; no voters:
09:13:37.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.800720 28222 raft_consensus.cc:2804] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:13:37.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.801072 28222 raft_consensus.cc:493] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:13:37.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.801383 28222 raft_consensus.cc:3060] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Advancing to term 1
09:13:37.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.808249 28222 raft_consensus.cc:515] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.812175 28144 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "8b5bb375b623431b90ededae66a9070a" candidate_uuid: "b03cd02817ea44cfa620f4c79f743171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" is_pre_election: true
09:13:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.812819 28143 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "8b5bb375b623431b90ededae66a9070a" candidate_uuid: "b03cd02817ea44cfa620f4c79f743171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c"
09:13:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.813097 28144 raft_consensus.cc:2468] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b03cd02817ea44cfa620f4c79f743171 in term 0.
09:13:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.813400 27996 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "8b5bb375b623431b90ededae66a9070a" candidate_uuid: "b03cd02817ea44cfa620f4c79f743171" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a968a863d78b44118abd134c71c37377"
09:13:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.811312 28222 leader_election.cc:290] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [CANDIDATE]: Term 1 election: Requested vote from peers a968a863d78b44118abd134c71c37377 (127.26.197.66:43885), a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67:45421)
09:13:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.813951 27996 raft_consensus.cc:3060] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Advancing to term 1
09:13:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.820602 27996 raft_consensus.cc:2468] T 8b5bb375b623431b90ededae66a9070a P a968a863d78b44118abd134c71c37377 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b03cd02817ea44cfa620f4c79f743171 in term 1.
09:13:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.821707 27768 leader_election.cc:304] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a968a863d78b44118abd134c71c37377, b03cd02817ea44cfa620f4c79f743171; no voters: a8d35d68bd544dd1bb30304d9ec69d5c
09:13:37.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.823259 28222 raft_consensus.cc:2804] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 1 FOLLOWER]: Leader election won for term 1
09:13:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.826249 28222 raft_consensus.cc:697] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [term 1 LEADER]: Becoming Leader. State: Replica: b03cd02817ea44cfa620f4c79f743171, State: Running, Role: LEADER
09:13:37.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.827382 28222 consensus_queue.cc:237] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:37.837864 27895 tablet.cc:2378] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:13:37.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.845094 27634 catalog_manager.cc:5654] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 reported cstate change: term changed from 0 to 1, leader changed from <none> to b03cd02817ea44cfa620f4c79f743171 (127.26.197.65). New cstate: current_term: 1 leader_uuid: "b03cd02817ea44cfa620f4c79f743171" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } health_report { overall_health: UNKNOWN } } }
09:13:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:37.856652 28044 tablet.cc:2378] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:13:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:37.876231 28192 tablet.cc:2378] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:13:37.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.985368 28221 raft_consensus.cc:493] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:13:37.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.985788 28221 raft_consensus.cc:515] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } }
09:13:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:37.988157 28221 leader_election.cc:290] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67:45421), b03cd02817ea44cfa620f4c79f743171 (127.26.197.65:45603)
09:13:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.000453 28144 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "51549a3ef4aa4d7597f571ac38fa192f" candidate_uuid: "a968a863d78b44118abd134c71c37377" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" is_pre_election: true
09:13:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.001065 28144 raft_consensus.cc:2468] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a968a863d78b44118abd134c71c37377 in term 0.
09:13:38.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.001199 27847 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "51549a3ef4aa4d7597f571ac38fa192f" candidate_uuid: "a968a863d78b44118abd134c71c37377" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b03cd02817ea44cfa620f4c79f743171" is_pre_election: true
09:13:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.001618 28223 raft_consensus.cc:493] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:13:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.001982 27847 raft_consensus.cc:2468] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a968a863d78b44118abd134c71c37377 in term 0.
09:13:38.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.002045 28223 raft_consensus.cc:515] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } }
09:13:38.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.002525 27919 leader_election.cc:304] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a8d35d68bd544dd1bb30304d9ec69d5c, a968a863d78b44118abd134c71c37377; no voters:
09:13:38.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.003422 28221 raft_consensus.cc:2804] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:13:38.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.003712 28221 raft_consensus.cc:493] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:13:38.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.004031 28221 raft_consensus.cc:3060] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Advancing to term 1
09:13:38.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.004956 28223 leader_election.cc:290] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a968a863d78b44118abd134c71c37377 (127.26.197.66:43885), b03cd02817ea44cfa620f4c79f743171 (127.26.197.65:45603)
09:13:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.009948 28221 raft_consensus.cc:515] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } }
09:13:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.012800 28144 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "51549a3ef4aa4d7597f571ac38fa192f" candidate_uuid: "a968a863d78b44118abd134c71c37377" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c"
09:13:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.013142 27847 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "51549a3ef4aa4d7597f571ac38fa192f" candidate_uuid: "a968a863d78b44118abd134c71c37377" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b03cd02817ea44cfa620f4c79f743171"
09:13:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.013407 28144 raft_consensus.cc:3060] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Advancing to term 1
09:13:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.013666 27847 raft_consensus.cc:3060] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Advancing to term 1
09:13:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.019279 27996 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "51549a3ef4aa4d7597f571ac38fa192f" candidate_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a968a863d78b44118abd134c71c37377" is_pre_election: true
09:13:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.020133 27996 raft_consensus.cc:2393] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a8d35d68bd544dd1bb30304d9ec69d5c in current term 1: Already voted for candidate a968a863d78b44118abd134c71c37377 in this term.
09:13:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.020319 28221 leader_election.cc:290] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [CANDIDATE]: Term 1 election: Requested vote from peers a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67:45421), b03cd02817ea44cfa620f4c79f743171 (127.26.197.65:45603)
09:13:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.019974 28144 raft_consensus.cc:2468] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a968a863d78b44118abd134c71c37377 in term 1.
09:13:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.021657 27919 leader_election.cc:304] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a8d35d68bd544dd1bb30304d9ec69d5c, a968a863d78b44118abd134c71c37377; no voters:
09:13:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.022420 28221 raft_consensus.cc:2804] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 1 FOLLOWER]: Leader election won for term 1
09:13:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.023314 27846 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "51549a3ef4aa4d7597f571ac38fa192f" candidate_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b03cd02817ea44cfa620f4c79f743171" is_pre_election: true
09:13:38.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.024695 28221 raft_consensus.cc:697] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [term 1 LEADER]: Becoming Leader. State: Replica: a968a863d78b44118abd134c71c37377, State: Running, Role: LEADER
09:13:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.026085 27847 raft_consensus.cc:2468] T 51549a3ef4aa4d7597f571ac38fa192f P b03cd02817ea44cfa620f4c79f743171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a968a863d78b44118abd134c71c37377 in term 1.
09:13:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.026023 28221 consensus_queue.cc:237] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } }
09:13:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.026454 28066 leader_election.cc:304] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a8d35d68bd544dd1bb30304d9ec69d5c; no voters: a968a863d78b44118abd134c71c37377, b03cd02817ea44cfa620f4c79f743171
09:13:38.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.027391 28223 raft_consensus.cc:2749] T 51549a3ef4aa4d7597f571ac38fa192f P a8d35d68bd544dd1bb30304d9ec69d5c [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
09:13:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.038239 27634 catalog_manager.cc:5654] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 reported cstate change: term changed from 0 to 1, leader changed from <none> to a968a863d78b44118abd134c71c37377 (127.26.197.66). New cstate: current_term: 1 leader_uuid: "a968a863d78b44118abd134c71c37377" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } health_report { overall_health: UNKNOWN } } }
09:13:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.404731 28144 raft_consensus.cc:3060] T 8b5bb375b623431b90ededae66a9070a P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Advancing to term 1
09:13:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.405836 28239 consensus_queue.cc:1048] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:13:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.411561 28239 consensus_queue.cc:1048] T 8b5bb375b623431b90ededae66a9070a P b03cd02817ea44cfa620f4c79f743171 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
09:13:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.517758 28221 consensus_queue.cc:1048] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:13:38.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.538659 28221 consensus_queue.cc:1048] T 51549a3ef4aa4d7597f571ac38fa192f P a968a863d78b44118abd134c71c37377 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:13:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.604920 27633 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45256:
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:13:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:13:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:13:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:38.607249 27633 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:13:38.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.642164 27827 tablet_service.cc:1505] Processing CreateTablet for tablet 57f6e1962acd4ed59809df3980adf3c4 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=ff3716932b0f4b31bdb20d5031b99373]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:13:38.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.643900 27827 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 57f6e1962acd4ed59809df3980adf3c4. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:38.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.644263 27976 tablet_service.cc:1505] Processing CreateTablet for tablet 57f6e1962acd4ed59809df3980adf3c4 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=ff3716932b0f4b31bdb20d5031b99373]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:13:38.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.645841 27976 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 57f6e1962acd4ed59809df3980adf3c4. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:38.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.645313 28123 tablet_service.cc:1505] Processing CreateTablet for tablet 57f6e1962acd4ed59809df3980adf3c4 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=ff3716932b0f4b31bdb20d5031b99373]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:13:38.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.646867 28123 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 57f6e1962acd4ed59809df3980adf3c4. 1 dirs total, 0 dirs full, 0 dirs failed
09:13:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.659704 28268 tablet_bootstrap.cc:492] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: Bootstrap starting.
09:13:38.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.664903 28270 tablet_bootstrap.cc:492] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: Bootstrap starting.
09:13:38.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.666885 28269 tablet_bootstrap.cc:492] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: Bootstrap starting.
09:13:38.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.667326 28268 tablet_bootstrap.cc:654] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: Neither blocks nor log segments found. Creating new log.
09:13:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.671092 28270 tablet_bootstrap.cc:654] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: Neither blocks nor log segments found. Creating new log.
09:13:38.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.671801 28269 tablet_bootstrap.cc:654] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: Neither blocks nor log segments found. Creating new log.
09:13:38.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.675683 28268 tablet_bootstrap.cc:492] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: No bootstrap required, opened a new log
09:13:38.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.676060 28268 ts_tablet_manager.cc:1403] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: Time spent bootstrapping tablet: real 0.017s user 0.010s sys 0.000s
09:13:38.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.676645 28270 tablet_bootstrap.cc:492] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: No bootstrap required, opened a new log
09:13:38.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.677053 28270 ts_tablet_manager.cc:1403] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: Time spent bootstrapping tablet: real 0.012s user 0.010s sys 0.000s
09:13:38.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.677359 28269 tablet_bootstrap.cc:492] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: No bootstrap required, opened a new log
09:13:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.677793 28269 ts_tablet_manager.cc:1403] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.006s
09:13:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.678226 28268 raft_consensus.cc:359] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.678718 28268 raft_consensus.cc:385] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:38.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.678977 28268 raft_consensus.cc:740] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8d35d68bd544dd1bb30304d9ec69d5c, State: Initialized, Role: FOLLOWER
09:13:38.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.680164 28268 consensus_queue.cc:260] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.681538 28223 tablet_replica.cc:442] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } }
09:13:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.682289 28223 tablet_replica.cc:445] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: This TxnStatusTablet replica's current role is: FOLLOWER
09:13:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.682551 28268 ts_tablet_manager.cc:1434] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: Time spent starting tablet: real 0.006s user 0.007s sys 0.000s
09:13:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.680773 28270 raft_consensus.cc:359] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.683151 28270 raft_consensus.cc:385] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.683467 28270 raft_consensus.cc:740] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a968a863d78b44118abd134c71c37377, State: Initialized, Role: FOLLOWER
09:13:38.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.684118 28270 consensus_queue.cc:260] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.680768 28269 raft_consensus.cc:359] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.686133 28269 raft_consensus.cc:385] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:13:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.686447 28269 raft_consensus.cc:740] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b03cd02817ea44cfa620f4c79f743171, State: Initialized, Role: FOLLOWER
09:13:38.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.687307 28269 consensus_queue.cc:260] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:38.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.688864 28241 tablet_replica.cc:442] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } }
09:13:38.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.689644 28241 tablet_replica.cc:445] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: This TxnStatusTablet replica's current role is: FOLLOWER
09:13:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.695060 28247 tablet_replica.cc:442] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } }
09:13:38.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.695838 28247 tablet_replica.cc:445] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: This TxnStatusTablet replica's current role is: FOLLOWER
09:13:38.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.696013 28270 ts_tablet_manager.cc:1434] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: Time spent starting tablet: real 0.019s user 0.008s sys 0.000s
09:13:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.707082 28269 ts_tablet_manager.cc:1434] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: Time spent starting tablet: real 0.029s user 0.006s sys 0.002s
09:13:38.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.718485 28223 raft_consensus.cc:493] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:13:38.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.718995 28223 raft_consensus.cc:515] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.720520 28223 leader_election.cc:290] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b03cd02817ea44cfa620f4c79f743171 (127.26.197.65:45603), a968a863d78b44118abd134c71c37377 (127.26.197.66:43885)
09:13:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.721323 27847 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "57f6e1962acd4ed59809df3980adf3c4" candidate_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b03cd02817ea44cfa620f4c79f743171" is_pre_election: true
09:13:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.721606 27996 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "57f6e1962acd4ed59809df3980adf3c4" candidate_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a968a863d78b44118abd134c71c37377" is_pre_election: true
09:13:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.721791 27847 raft_consensus.cc:2468] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a8d35d68bd544dd1bb30304d9ec69d5c in term 0.
09:13:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.722151 27996 raft_consensus.cc:2468] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a8d35d68bd544dd1bb30304d9ec69d5c in term 0.
09:13:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.722568 28066 leader_election.cc:304] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a8d35d68bd544dd1bb30304d9ec69d5c, b03cd02817ea44cfa620f4c79f743171; no voters:
09:13:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.723234 28223 raft_consensus.cc:2804] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Leader pre-election won for term 1
09:13:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.723503 28223 raft_consensus.cc:493] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:13:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.723769 28223 raft_consensus.cc:3060] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 0 FOLLOWER]: Advancing to term 1
09:13:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.727869 28223 raft_consensus.cc:515] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.729382 28223 leader_election.cc:290] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [CANDIDATE]: Term 1 election: Requested vote from peers b03cd02817ea44cfa620f4c79f743171 (127.26.197.65:45603), a968a863d78b44118abd134c71c37377 (127.26.197.66:43885)
09:13:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.730173 27847 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "57f6e1962acd4ed59809df3980adf3c4" candidate_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b03cd02817ea44cfa620f4c79f743171"
09:13:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.730401 27996 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "57f6e1962acd4ed59809df3980adf3c4" candidate_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a968a863d78b44118abd134c71c37377"
09:13:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.730652 27847 raft_consensus.cc:3060] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171 [term 0 FOLLOWER]: Advancing to term 1
09:13:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.730989 27996 raft_consensus.cc:3060] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377 [term 0 FOLLOWER]: Advancing to term 1
09:13:38.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.735082 27847 raft_consensus.cc:2468] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a8d35d68bd544dd1bb30304d9ec69d5c in term 1.
09:13:38.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.735399 27996 raft_consensus.cc:2468] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a8d35d68bd544dd1bb30304d9ec69d5c in term 1.
09:13:38.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.736025 28066 leader_election.cc:304] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a8d35d68bd544dd1bb30304d9ec69d5c, b03cd02817ea44cfa620f4c79f743171; no voters:
09:13:38.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.736829 28223 raft_consensus.cc:2804] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 1 FOLLOWER]: Leader election won for term 1
09:13:38.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.738534 28223 raft_consensus.cc:697] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [term 1 LEADER]: Becoming Leader. State: Replica: a8d35d68bd544dd1bb30304d9ec69d5c, State: Running, Role: LEADER
09:13:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.739537 28223 consensus_queue.cc:237] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } }
09:13:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.747967 27633 catalog_manager.cc:5654] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c reported cstate change: term changed from 0 to 1, leader changed from <none> to a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67). New cstate: current_term: 1 leader_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } health_report { overall_health: HEALTHY } } }
09:13:38.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.750253 28274 tablet_replica.cc:442] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: TxnStatusTablet state changed. Reason: New leader a8d35d68bd544dd1bb30304d9ec69d5c. Latest consensus state: current_term: 1 leader_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } }
09:13:38.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.750820 28274 tablet_replica.cc:445] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: This TxnStatusTablet replica's current role is: LEADER
09:13:38.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.754288 28276 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
09:13:38.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.754846 28276 txn_status_manager.cc:930] Loading transaction status metadata into memory...
09:13:38.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.819044 27996 raft_consensus.cc:1275] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377 [term 1 FOLLOWER]: Refusing update from remote peer a8d35d68bd544dd1bb30304d9ec69d5c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:13:38.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.819564 27847 raft_consensus.cc:1275] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171 [term 1 FOLLOWER]: Refusing update from remote peer a8d35d68bd544dd1bb30304d9ec69d5c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:13:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.820864 28223 consensus_queue.cc:1048] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:13:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.821619 28274 consensus_queue.cc:1048] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c [LEADER]: Connected to new peer: Peer: permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:13:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.848315 28247 tablet_replica.cc:442] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: TxnStatusTablet state changed. Reason: New leader a8d35d68bd544dd1bb30304d9ec69d5c. Latest consensus state: current_term: 1 leader_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } }
09:13:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.848989 28247 tablet_replica.cc:445] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: This TxnStatusTablet replica's current role is: FOLLOWER
09:13:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.851523 28245 mvcc.cc:204] Tried to move back new op lower bound from 7231968128267501568 to 7231968127963996160. Current Snapshot: MvccSnapshot[applied={T|T < 7231968128267501568}]
09:13:38.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.856173 28244 mvcc.cc:204] Tried to move back new op lower bound from 7231968128267501568 to 7231968127963996160. Current Snapshot: MvccSnapshot[applied={T|T < 7231968128267501568}]
09:13:38.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.866493 28241 tablet_replica.cc:442] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: TxnStatusTablet state changed. Reason: New leader a8d35d68bd544dd1bb30304d9ec69d5c. Latest consensus state: current_term: 1 leader_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } }
09:13:38.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.867264 28241 tablet_replica.cc:445] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: This TxnStatusTablet replica's current role is: FOLLOWER
09:13:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.886533 28242 mvcc.cc:204] Tried to move back new op lower bound from 7231968128267501568 to 7231968127963996160. Current Snapshot: MvccSnapshot[applied={T|T < 7231968128267501568}]
09:13:38.901 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
09:13:38.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.894798 28274 tablet_replica.cc:442] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } }
09:13:38.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.895515 28274 tablet_replica.cc:445] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: This TxnStatusTablet replica's current role is: LEADER
09:13:38.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.898636 28274 tablet_replica.cc:442] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } }
09:13:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.899286 28274 tablet_replica.cc:445] T 57f6e1962acd4ed59809df3980adf3c4 P a8d35d68bd544dd1bb30304d9ec69d5c: This TxnStatusTablet replica's current role is: LEADER
09:13:38.904 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 03eaf326a2654e0dab58aa42e3c28b4a
09:13:38.905 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.197.124:35671
09:13:38.905 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
09:13:38.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.906378 27413 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu with pid 27590
09:13:38.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.910823 28247 tablet_replica.cc:442] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } }
09:13:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.911484 28247 tablet_replica.cc:445] T 57f6e1962acd4ed59809df3980adf3c4 P a968a863d78b44118abd134c71c37377: This TxnStatusTablet replica's current role is: FOLLOWER
09:13:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.919286 28241 tablet_replica.cc:442] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" member_type: VOTER last_known_addr { host: "127.26.197.65" port: 45603 } } peers { permanent_uuid: "a968a863d78b44118abd134c71c37377" member_type: VOTER last_known_addr { host: "127.26.197.66" port: 43885 } } peers { permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" member_type: VOTER last_known_addr { host: "127.26.197.67" port: 45421 } } }
09:13:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:38.919955 28241 tablet_replica.cc:445] T 57f6e1962acd4ed59809df3980adf3c4 P b03cd02817ea44cfa620f4c79f743171: This TxnStatusTablet replica's current role is: FOLLOWER
09:13:38.945 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.26.197.124:35671(127.26.197.124:35671)] handling channelInactive
09:13:38.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.197.124:35671(127.26.197.124:35671)] cleaning up while in state READY due to: connection closed
09:13:38.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:38.944608 27919 connection.cc:537] client connection to 127.26.197.124:35671 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:13:38.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:38.945443 28042 heartbeater.cc:646] Failed to heartbeat to 127.26.197.124:35671 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:13:38.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:38.945400 28190 heartbeater.cc:646] Failed to heartbeat to 127.26.197.124:35671 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.26.197.124:35671 (error 108)
09:13:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:38.948576 27770 connection.cc:537] client connection to 127.26.197.124:35671 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:13:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:38.950086 27893 heartbeater.cc:646] Failed to heartbeat to 127.26.197.124:35671 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:13:38.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:38.953759 28285 negotiation.cc:337] Failed RPC negotiation. Trace:
09:13:38.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:38.946564 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.26.197.124:35671 (local address 127.26.197.67:49595)
09:13:38.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:38.952305 (+ 5741us) negotiation.cc:107] Waiting for socket to connect
09:13:38.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:38.952581 (+ 276us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection reset by peer (error 104)
09:13:38.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":5095,"threads_started":1}
09:13:38.955 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:13:39.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.160356 28288 raft_consensus.cc:493] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 1cd9f73088cd4a8fa0cdc72a8dd908ba)
09:13:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.160805 28288 raft_consensus.cc:515] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.162592 28288 leader_election.cc:290] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303), 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671)
09:13:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.163671 27466 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80b3786ba7c94e0d96350abc26a3bee6" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "3a70b898276748bba9d75a67edd84c5d" is_pre_election: true
09:13:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:39.165753 27529 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111) [suppressed 6 similar messages]
09:13:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:39.169184 27529 leader_election.cc:336] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671): Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.169538 27529 leader_election.cc:304] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 80b3786ba7c94e0d96350abc26a3bee6; no voters: 1cd9f73088cd4a8fa0cdc72a8dd908ba, 3a70b898276748bba9d75a67edd84c5d
09:13:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.170126 28288 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
09:13:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.185894 28291 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 1cd9f73088cd4a8fa0cdc72a8dd908ba)
09:13:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.186331 28291 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:39.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.188369 28291 leader_election.cc:290] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287), 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671)
09:13:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.189342 27552 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3a70b898276748bba9d75a67edd84c5d" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "80b3786ba7c94e0d96350abc26a3bee6" is_pre_election: true
09:13:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.189990 27552 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3a70b898276748bba9d75a67edd84c5d in term 1.
09:13:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:39.191083 27444 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111) [suppressed 13 similar messages]
09:13:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.191213 27443 leader_election.cc:304] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3a70b898276748bba9d75a67edd84c5d, 80b3786ba7c94e0d96350abc26a3bee6; no voters:
09:13:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.191926 28291 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 1 FOLLOWER]: Leader pre-election won for term 2
09:13:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.192181 28291 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 1 FOLLOWER]: Starting leader election (detected failure of leader 1cd9f73088cd4a8fa0cdc72a8dd908ba)
09:13:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.192456 28291 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 1 FOLLOWER]: Advancing to term 2
09:13:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:39.198685 27444 leader_election.cc:336] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671): Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.198585 28291 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.200331 28291 leader_election.cc:290] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [CANDIDATE]: Term 2 election: Requested vote from peers 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287), 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671)
09:13:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.201035 27552 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3a70b898276748bba9d75a67edd84c5d" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "80b3786ba7c94e0d96350abc26a3bee6"
09:13:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.201567 27552 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 1 FOLLOWER]: Advancing to term 2
09:13:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:39.204345 27444 leader_election.cc:336] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671): Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111)
09:13:39.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.207950 27552 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3a70b898276748bba9d75a67edd84c5d in term 2.
09:13:39.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.208843 27443 leader_election.cc:304] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3a70b898276748bba9d75a67edd84c5d, 80b3786ba7c94e0d96350abc26a3bee6; no voters: 1cd9f73088cd4a8fa0cdc72a8dd908ba
09:13:39.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.209434 28291 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 2 FOLLOWER]: Leader election won for term 2
09:13:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.210695 28291 raft_consensus.cc:697] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 2 LEADER]: Becoming Leader. State: Replica: 3a70b898276748bba9d75a67edd84c5d, State: Running, Role: LEADER
09:13:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.211511 28291 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.214591 28294 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3a70b898276748bba9d75a67edd84c5d. Latest consensus state: current_term: 2 leader_uuid: "3a70b898276748bba9d75a67edd84c5d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.215452 28294 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: This master's current role is: LEADER
09:13:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.217129 28295 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:13:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.223930 28295 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a]
09:13:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.225068 28295 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=ff3716932b0f4b31bdb20d5031b99373]
09:13:39.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.229801 28295 tablet_loader.cc:96] loaded metadata for tablet 51549a3ef4aa4d7597f571ac38fa192f (table txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a])
09:13:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.230800 28295 tablet_loader.cc:96] loaded metadata for tablet 57f6e1962acd4ed59809df3980adf3c4 (table kudu_system.kudu_transactions [id=ff3716932b0f4b31bdb20d5031b99373])
09:13:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.231695 28295 tablet_loader.cc:96] loaded metadata for tablet 8b5bb375b623431b90ededae66a9070a (table txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a])
09:13:39.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.232661 28295 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:13:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.235620 28295 catalog_manager.cc:1269] Loaded cluster ID: 812ace05a5054ff79e14cf188ae3b8f5
09:13:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.235792 28295 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:13:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.239447 28295 catalog_manager.cc:1514] Loading token signing keys...
09:13:39.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.242275 28295 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: Loaded TSK: 0
09:13:39.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.243336 28295 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:13:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.339666 27552 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 2 FOLLOWER]: Refusing update from remote peer 3a70b898276748bba9d75a67edd84c5d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
09:13:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:39.341091 27444 consensus_peers.cc:597] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d -> Peer 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671): Couldn't send request to peer 1cd9f73088cd4a8fa0cdc72a8dd908ba. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:13:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.341566 28294 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [LEADER]: Connected to new peer: Peer: permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
09:13:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.345786 28288 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3a70b898276748bba9d75a67edd84c5d. Latest consensus state: current_term: 2 leader_uuid: "3a70b898276748bba9d75a67edd84c5d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.346557 28288 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: This master's current role is: FOLLOWER
09:13:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.349702 28291 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "3a70b898276748bba9d75a67edd84c5d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.350435 28291 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: This master's current role is: LEADER
09:13:39.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.353232 28288 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "3a70b898276748bba9d75a67edd84c5d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:39.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.353745 28288 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: This master's current role is: FOLLOWER
09:13:39.413 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.124:35671(127.26.197.124:35671)] connecting to peer
09:13:39.415 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.197.124:35671(127.26.197.124:35671): Connection refused: /127.26.197.124:35671
09:13:39.415 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.197.124:35671(127.26.197.124:35671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.124:35671(127.26.197.124:35671): Connection refused: /127.26.197.124:35671
09:13:39.415 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.26.197.124:35671(127.26.197.124:35671) for tablet Kudu Master: Failed to connect to peer master-127.26.197.124:35671(127.26.197.124:35671): Connection refused: /127.26.197.124:35671
09:13:39.432 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.124:35671(127.26.197.124:35671)] connecting to peer
09:13:39.434 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.197.124:35671(127.26.197.124:35671): Connection refused: /127.26.197.124:35671
09:13:39.434 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.124:35671(127.26.197.124:35671)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.124:35671(127.26.197.124:35671): Connection refused: /127.26.197.124:35671
09:13:39.434 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.124:35671: Failed to connect to peer master-127.26.197.124:35671(127.26.197.124:35671): Connection refused: /127.26.197.124:35671
09:13:39.437 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:13:39.438 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=25), Traces: [0ms] sending RPC to server master-127.26.197.124:35671, [2ms] received response from server master-127.26.197.124:35671: Network error: Failed to connect to peer master-127.26.197.124:35671(127.26.197.124:35671): Connection refused: /127.26.197.124:35671, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.26.197.124:35671(127.26.197.124:35671): Connection refused: /127.26.197.124:35671, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [21ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: Network error: Failed to connect to peer master-127.26.197.124:35671(127.26.197.124:35671): Connection refused: /127.26.197.124:35671, [22ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, deferred=Deferred@1640748267(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/331225083@1912d64d, errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/477708768@2e469add)) after lookup
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:39.487016 27458 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50156:
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:13:39.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:39.512888 27458 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:13:39.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:39.821655 27444 consensus_peers.cc:597] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d -> Peer 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671): Couldn't send request to peer 1cd9f73088cd4a8fa0cdc72a8dd908ba. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:13:40.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:40.366425 27444 consensus_peers.cc:597] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d -> Peer 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671): Couldn't send request to peer 1cd9f73088cd4a8fa0cdc72a8dd908ba. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:13:40.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:40.843569 27444 consensus_peers.cc:597] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d -> Peer 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671): Couldn't send request to peer 1cd9f73088cd4a8fa0cdc72a8dd908ba. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:13:40.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:40.913061 28191 heartbeater.cc:499] Master 127.26.197.126:42303 was elected leader, sending a full tablet report...
09:13:40.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:40.929917 27894 heartbeater.cc:499] Master 127.26.197.126:42303 was elected leader, sending a full tablet report...
09:13:40.962 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 51549a3ef4aa4d7597f571ac38fa192f for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
09:13:40.963 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 8b5bb375b623431b90ededae66a9070a for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
09:13:40.963 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=51549a3ef4aa4d7597f571ac38fa192f@[a8d35d68bd544dd1bb30304d9ec69d5c(127.26.197.67:45421),a968a863d78b44118abd134c71c37377(127.26.197.66:43885)[L],b03cd02817ea44cfa620f4c79f743171(127.26.197.65:45603)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=8b5bb375b623431b90ededae66a9070a@[a8d35d68bd544dd1bb30304d9ec69d5c(127.26.197.67:45421),a968a863d78b44118abd134c71c37377(127.26.197.66:43885),b03cd02817ea44cfa620f4c79f743171(127.26.197.65:45603)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
09:13:40.964 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.197.126:42303, [3ms] Sub RPC GetTableLocations: received response from server master-127.26.197.126:42303: OK, deferred=null) after lookup
09:13:40.964 [DEBUG - kudu-nio-0] (Connection.java:748) [peer a968a863d78b44118abd134c71c37377(127.26.197.66:43885)] connecting to peer
09:13:40.965 [DEBUG - kudu-nio-1] (Connection.java:757) [peer a968a863d78b44118abd134c71c37377(127.26.197.66:43885)] Successfully connected to peer
09:13:40.979 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x5813c65c, L:/127.0.0.1:49722 - R:/127.26.197.66:43885] using TOKEN/null
09:13:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:40.985555 27936 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.26.197.124:35671)
09:13:41.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:40.997169 28043 heartbeater.cc:499] Master 127.26.197.126:42303 was elected leader, sending a full tablet report...
09:13:41.079 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.197.124:35671
09:13:41.080 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:41.081687 27413 external_mini_cluster.cc:1366] Running /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/logs
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data/info.pb
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.124:35671
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.124
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35399
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34909/jwks.json
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:13:41.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:13:41.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.293370 27444 consensus_peers.cc:597] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d -> Peer 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671): Couldn't send request to peer 1cd9f73088cd4a8fa0cdc72a8dd908ba. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
09:13:41.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.396631 28340 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:41.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.397218 28340 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:41.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.397728 28340 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:41.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.430532 28340 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:13:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.430881 28340 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:13:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.431061 28340 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.431262 28340 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:13:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.431456 28340 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:13:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.431645 28340 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:13:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.431782 28340 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34909/jwks.json
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:41.470619 28340 master_runner.cc:386] Master server non-default flags:
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.124:35671
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34909/jwks.json
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data/info.pb
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.124
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35399
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/logs
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 08:43:20 UTC on 5fd53c4cbb9d
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9493
09:13:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:41.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:41.472002 28340 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:41.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:41.473534 28340 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.484243 28346 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:41.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.779529 27444 consensus_peers.cc:597] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d -> Peer 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671): Couldn't send request to peer 1cd9f73088cd4a8fa0cdc72a8dd908ba. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
09:13:42.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:42.305886 27444 consensus_peers.cc:597] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d -> Peer 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671): Couldn't send request to peer 1cd9f73088cd4a8fa0cdc72a8dd908ba. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.124:35671: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
09:13:42.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:41.484774 28347 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:42.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:42.664765 28349 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:42.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:42.667328 28348 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1178 milliseconds
09:13:42.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.667505 28340 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.668802 28340 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:42.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.671464 28340 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:42.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.672845 28340 hybrid_clock.cc:648] HybridClock initialized: now 1765617222672811 us; error 39 us; skew 500 ppm
09:13:42.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.673748 28340 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:42.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.680238 28340 webserver.cc:492] Webserver started at http://127.26.197.124:35399/ using document root <none> and password file <none>
09:13:42.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.681219 28340 fs_manager.cc:362] Metadata directory not provided
09:13:42.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.681458 28340 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:42.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.690430 28340 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
09:13:42.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.695317 28362 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.696523 28340 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
09:13:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.696825 28340 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data,/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal
09:13:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba"
09:13:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:29 on dist-test-slave-k5d4"
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.698732 28340 fs_report.cc:389] FS layout report
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/wal
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data/data
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.749588 28340 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.751487 28340 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.752038 28340 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:42.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.843025 28340 rpc_server.cc:307] RPC server started. Bound to: 127.26.197.124:35671
09:13:42.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.843089 28423 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.197.124:35671 every 8 connection(s)
09:13:42.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.846351 28340 server_base.cc:1179] Dumped server information to /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-2/data/info.pb
09:13:42.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.848940 27413 external_mini_cluster.cc:1428] Started /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu as pid 28340
09:13:42.849 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:13:42.849 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.197.126:42303
09:13:42.850 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
09:13:42.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.850383 27413 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu with pid 27425
09:13:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.862875 28425 sys_catalog.cc:263] Verifying existing consensus state
09:13:42.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.870308 28425 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: Bootstrap starting.
09:13:42.895 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.26.197.126:42303(127.26.197.126:42303)] handling channelInactive
09:13:42.895 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.126:42303(127.26.197.126:42303)] cleaning up while in state READY due to: connection closed
09:13:42.896 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:13:42.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:42.898494 28424 negotiation.cc:337] Failed RPC negotiation. Trace:
09:13:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:42.844529 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:49164 (local address 127.26.197.124:35671)
09:13:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:42.847315 (+ 2786us) server_negotiation.cc:207] Beginning negotiation
09:13:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:42.847353 (+ 38us) server_negotiation.cc:400] Waiting for connection header
09:13:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:42.847435 (+ 82us) server_negotiation.cc:408] Connection header received
09:13:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:42.854297 (+ 6862us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
09:13:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:42.854361 (+ 64us) server_negotiation.cc:462] Received NEGOTIATE request from client
09:13:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:42.855104 (+ 743us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
09:13:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:42.855607 (+ 503us) server_negotiation.cc:227] Negotiated authn=SASL
09:13:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:42.894089 (+ 38482us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49164: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:13:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1911,"thread_start_us":1459,"threads_started":1}
09:13:42.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.921344 28425 log.cc:826] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: Log is configured to *not* fsync() on all Append() calls
09:13:42.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.942915 28190 heartbeater.cc:344] Connected to a master server at 127.26.197.124:35671
09:13:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.959908 28425 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:13:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.960685 28425 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: Bootstrap complete.
09:13:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.971065 27893 heartbeater.cc:344] Connected to a master server at 127.26.197.124:35671
09:13:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.982823 28425 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.985127 28425 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1cd9f73088cd4a8fa0cdc72a8dd908ba, State: Initialized, Role: FOLLOWER
09:13:42.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.985885 28425 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:42.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.988283 28431 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.988888 28431 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: This master's current role is: FOLLOWER
09:13:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:42.989681 28425 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: configured and running, proceeding with master startup.
09:13:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.016848 28442 catalog_manager.cc:1269] Loaded cluster ID: 812ace05a5054ff79e14cf188ae3b8f5
09:13:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.017132 28442 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: loading cluster ID for follower catalog manager: success
09:13:43.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.025725 28442 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: acquiring CA information for follower catalog manager: success
09:13:43.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.032182 28442 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:13:43.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.012956 28340 master_runner.cc:186] Error getting master registration for 127.26.197.126:42303: Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111),
09:13:43.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.047333 28340 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:13:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.048277 28042 heartbeater.cc:344] Connected to a master server at 127.26.197.124:35671
09:13:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.053988 28379 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a968a863d78b44118abd134c71c37377" instance_seqno: 1765617215062400) as {username='slave'} at 127.26.197.66:43193; Asking this server to re-register.
09:13:43.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.055351 28042 heartbeater.cc:461] Registering TS with master...
09:13:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.058483 28379 ts_manager.cc:194] Registered new tserver with Master: a968a863d78b44118abd134c71c37377 (127.26.197.66:43885)
09:13:43.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.069594 28447 raft_consensus.cc:493] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 3a70b898276748bba9d75a67edd84c5d)
09:13:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.069962 28447 raft_consensus.cc:515] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:43.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.071743 28447 leader_election.cc:290] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303), 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671)
09:13:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:43.077268 27530 leader_election.cc:336] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111)
09:13:43.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.084729 28389 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80b3786ba7c94e0d96350abc26a3bee6" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" is_pre_election: true
09:13:43.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.085592 28389 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 80b3786ba7c94e0d96350abc26a3bee6 in term 1.
09:13:43.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.086632 27529 leader_election.cc:304] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1cd9f73088cd4a8fa0cdc72a8dd908ba, 80b3786ba7c94e0d96350abc26a3bee6; no voters: 3a70b898276748bba9d75a67edd84c5d
09:13:43.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.087397 28447 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 2 FOLLOWER]: Leader pre-election won for term 3
09:13:43.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.087714 28447 raft_consensus.cc:493] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 3a70b898276748bba9d75a67edd84c5d)
09:13:43.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.088202 28447 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 2 FOLLOWER]: Advancing to term 3
09:13:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.094658 28447 raft_consensus.cc:515] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:43.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.096616 28447 leader_election.cc:290] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [CANDIDATE]: Term 3 election: Requested vote from peers 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303), 1cd9f73088cd4a8fa0cdc72a8dd908ba (127.26.197.124:35671)
09:13:43.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.097718 28389 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80b3786ba7c94e0d96350abc26a3bee6" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba"
09:13:43.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.098204 28389 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 1 FOLLOWER]: Advancing to term 3
09:13:43.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:43.100792 27530 leader_election.cc:336] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111)
09:13:43.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.108896 28389 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 80b3786ba7c94e0d96350abc26a3bee6 in term 3.
09:13:43.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.110064 27529 leader_election.cc:304] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1cd9f73088cd4a8fa0cdc72a8dd908ba, 80b3786ba7c94e0d96350abc26a3bee6; no voters: 3a70b898276748bba9d75a67edd84c5d
09:13:43.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.110888 28447 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 3 FOLLOWER]: Leader election won for term 3
09:13:43.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.112275 28447 raft_consensus.cc:697] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [term 3 LEADER]: Becoming Leader. State: Replica: 80b3786ba7c94e0d96350abc26a3bee6, State: Running, Role: LEADER
09:13:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.113235 28447 consensus_queue.cc:237] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.117668 28451 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 80b3786ba7c94e0d96350abc26a3bee6. Latest consensus state: current_term: 3 leader_uuid: "80b3786ba7c94e0d96350abc26a3bee6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.118388 28451 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: This master's current role is: LEADER
09:13:43.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.121488 28453 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:13:43.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.131158 28453 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a]
09:13:43.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.133136 28453 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=ff3716932b0f4b31bdb20d5031b99373]
09:13:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.140228 28453 tablet_loader.cc:96] loaded metadata for tablet 51549a3ef4aa4d7597f571ac38fa192f (table txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a])
09:13:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.141655 28453 tablet_loader.cc:96] loaded metadata for tablet 57f6e1962acd4ed59809df3980adf3c4 (table kudu_system.kudu_transactions [id=ff3716932b0f4b31bdb20d5031b99373])
09:13:43.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.143079 28453 tablet_loader.cc:96] loaded metadata for tablet 8b5bb375b623431b90ededae66a9070a (table txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a])
09:13:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.144551 28453 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:13:43.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.149039 28453 catalog_manager.cc:1269] Loaded cluster ID: 812ace05a5054ff79e14cf188ae3b8f5
09:13:43.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.149298 28453 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:13:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.153123 28453 catalog_manager.cc:1514] Loading token signing keys...
09:13:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.156726 28453 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6: Loaded TSK: 0
09:13:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.158231 28453 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:13:43.172 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.126:42303(127.26.197.126:42303)] connecting to peer
09:13:43.174 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.197.126:42303(127.26.197.126:42303): Connection refused: /127.26.197.126:42303
09:13:43.174 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.126:42303(127.26.197.126:42303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.126:42303(127.26.197.126:42303): Connection refused: /127.26.197.126:42303
09:13:43.174 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.26.197.126:42303(127.26.197.126:42303) for tablet Kudu Master: Failed to connect to peer master-127.26.197.126:42303(127.26.197.126:42303): Connection refused: /127.26.197.126:42303
09:13:43.192 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.124:35671(127.26.197.124:35671)] connecting to peer
09:13:43.193 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.126:42303(127.26.197.126:42303)] connecting to peer
09:13:43.193 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.26.197.124:35671(127.26.197.124:35671)] Successfully connected to peer
09:13:43.194 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.197.126:42303(127.26.197.126:42303): Connection refused: /127.26.197.126:42303
09:13:43.194 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.126:42303(127.26.197.126:42303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.126:42303(127.26.197.126:42303): Connection refused: /127.26.197.126:42303
09:13:43.196 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.126:42303: Failed to connect to peer master-127.26.197.126:42303(127.26.197.126:42303): Connection refused: /127.26.197.126:42303
09:13:43.201 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:13:43.202 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=29), Traces: [0ms] sending RPC to server master-127.26.197.126:42303, [2ms] received response from server master-127.26.197.126:42303: Network error: Failed to connect to peer master-127.26.197.126:42303(127.26.197.126:42303): Connection refused: /127.26.197.126:42303, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.26.197.126:42303(127.26.197.126:42303): Connection refused: /127.26.197.126:42303, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [22ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: Network error: Failed to connect to peer master-127.26.197.126:42303(127.26.197.126:42303): Connection refused: /127.26.197.126:42303, [28ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: OK, deferred=Deferred@1728331628(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/331225083@2af392fd, errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/477708768@716efe67)) after lookup
09:13:43.221 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xdeb45859, L:/127.0.0.1:49176 - R:/127.26.197.124:35671] using TOKEN/null
09:13:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.242414 28389 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 3 FOLLOWER]: Refusing update from remote peer 80b3786ba7c94e0d96350abc26a3bee6: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
09:13:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:43.243007 27530 consensus_peers.cc:597] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 -> Peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Couldn't send request to peer 3a70b898276748bba9d75a67edd84c5d. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:13:43.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.245744 28451 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
09:13:43.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.260476 28466 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "80b3786ba7c94e0d96350abc26a3bee6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.262322 27543 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44572:
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:43.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:13:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.262418 28466 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: This master's current role is: FOLLOWER
09:13:43.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:43.269071 27543 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:13:43.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.268975 28431 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: SysCatalogTable state changed. Reason: New leader 80b3786ba7c94e0d96350abc26a3bee6. Latest consensus state: current_term: 3 leader_uuid: "80b3786ba7c94e0d96350abc26a3bee6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:43.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.269812 28431 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: This master's current role is: FOLLOWER
09:13:43.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.272521 28447 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "80b3786ba7c94e0d96350abc26a3bee6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:43.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.273232 28447 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 [sys.catalog]: This master's current role is: LEADER
09:13:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.273407 28431 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "80b3786ba7c94e0d96350abc26a3bee6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.274040 28431 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: This master's current role is: FOLLOWER
09:13:43.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:43.753258 27530 consensus_peers.cc:597] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 -> Peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Couldn't send request to peer 3a70b898276748bba9d75a67edd84c5d. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:13:43.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.952301 28379 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" instance_seqno: 1765617217305234) as {username='slave'} at 127.26.197.67:35511; Asking this server to re-register.
09:13:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.953719 28190 heartbeater.cc:461] Registering TS with master...
09:13:43.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.957037 28379 ts_manager.cc:194] Registered new tserver with Master: a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67:45421)
09:13:43.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.978217 28379 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" instance_seqno: 1765617212783843) as {username='slave'} at 127.26.197.65:46663; Asking this server to re-register.
09:13:43.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.979499 27893 heartbeater.cc:461] Registering TS with master...
09:13:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:43.982308 28379 ts_manager.cc:194] Registered new tserver with Master: b03cd02817ea44cfa620f4c79f743171 (127.26.197.65:45603)
09:13:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:44.220331 27530 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111) [suppressed 12 similar messages]
09:13:44.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:44.222559 27530 consensus_peers.cc:597] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 -> Peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Couldn't send request to peer 3a70b898276748bba9d75a67edd84c5d. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:13:44.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:44.751595 27530 consensus_peers.cc:597] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 -> Peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Couldn't send request to peer 3a70b898276748bba9d75a67edd84c5d. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:13:44.902 [DEBUG - Time-limited test] (Connection.java:748) [peer b03cd02817ea44cfa620f4c79f743171(127.26.197.65:45603)] connecting to peer
09:13:44.903 [DEBUG - kudu-nio-1] (Connection.java:757) [peer b03cd02817ea44cfa620f4c79f743171(127.26.197.65:45603)] Successfully connected to peer
09:13:44.918 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x95f8b38c, L:/127.0.0.1:48044 - R:/127.26.197.65:45603] using TOKEN/null
09:13:44.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:44.934024 27790 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.26.197.124:35671)
09:13:44.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:44.942117 28189 heartbeater.cc:499] Master 127.26.197.125:37287 was elected leader, sending a full tablet report...
09:13:44.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:44.951498 27892 heartbeater.cc:499] Master 127.26.197.125:37287 was elected leader, sending a full tablet report...
09:13:45.007 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.197.126:42303
09:13:45.007 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
09:13:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:45.009424 27413 external_mini_cluster.cc:1366] Running /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu
09:13:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal
09:13:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data
09:13:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:13:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:13:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:13:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/logs
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data/info.pb
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.126:42303
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.126
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37079
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34909/jwks.json
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:13:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:13:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:45.027830 28041 heartbeater.cc:499] Master 127.26.197.125:37287 was elected leader, sending a full tablet report...
09:13:45.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.245731 27530 consensus_peers.cc:597] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 -> Peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Couldn't send request to peer 3a70b898276748bba9d75a67edd84c5d. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
09:13:45.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.359656 28498 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:13:45.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.360435 28498 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.361320 28498 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:13:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.415187 28498 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:13:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.415540 28498 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:13:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.415797 28498 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:13:45.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.416062 28498 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:13:45.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.416343 28498 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:13:45.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.416612 28498 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:13:45.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.416823 28498 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34909/jwks.json
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:45.465180 28498 master_runner.cc:386] Master server non-default flags:
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.197.84:45573
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.197.126:42303,127.26.197.125:37287,127.26.197.124:35671
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:13:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.197.126:42303
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34909/jwks.json
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data/info.pb
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.197.126
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37079
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/logs
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:13:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:13:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:13:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
09:13:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:13:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 08:43:20 UTC on 5fd53c4cbb9d
09:13:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9493
09:13:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:13:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:45.466840 28498 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:13:45.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:45.468585 28498 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:13:45.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.480363 28506 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:45.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.746006 27530 consensus_peers.cc:597] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 -> Peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Couldn't send request to peer 3a70b898276748bba9d75a67edd84c5d. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
09:13:46.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:46.226665 27530 consensus_peers.cc:597] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 -> Peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Couldn't send request to peer 3a70b898276748bba9d75a67edd84c5d. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
09:13:46.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:45.481781 28507 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:46.708338 28509 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:13:46.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:46.711172 28508 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1225 milliseconds
09:13:46.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.711396 28498 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:13:46.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.712772 28498 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:13:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.715668 28498 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:13:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.717061 28498 hybrid_clock.cc:648] HybridClock initialized: now 1765617226717017 us; error 47 us; skew 500 ppm
09:13:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.718042 28498 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:13:46.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.724646 28498 webserver.cc:492] Webserver started at http://127.26.197.126:37079/ using document root <none> and password file <none>
09:13:46.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.725637 28498 fs_manager.cc:362] Metadata directory not provided
09:13:46.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.725881 28498 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:13:46.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.734380 28498 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
09:13:46.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.739807 28525 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:13:46.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.741264 28498 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
09:13:46.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:46.741739 27530 consensus_peers.cc:597] T 00000000000000000000000000000000 P 80b3786ba7c94e0d96350abc26a3bee6 -> Peer 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303): Couldn't send request to peer 3a70b898276748bba9d75a67edd84c5d. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.126:42303: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
09:13:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.741650 28498 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data,/tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal
09:13:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a70b898276748bba9d75a67edd84c5d"
09:13:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 09:13:25 on dist-test-slave-k5d4"
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.743841 28498 fs_report.cc:389] FS layout report
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/wal
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data/data
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:13:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:13:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:13:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:13:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:13:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:13:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:13:46.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.806399 28498 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:13:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.808075 28498 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:13:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.808609 28498 kserver.cc:163] Server-wide thread pool size limit: 3276
09:13:46.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.913442 28498 rpc_server.cc:307] RPC server started. Bound to: 127.26.197.126:42303
09:13:46.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.913486 28586 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.197.126:42303 every 8 connection(s)
09:13:46.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.916453 28498 server_base.cc:1179] Dumped server information to /tmp/dist-test-task0xFmrh/test-tmp/mini-kudu-cluster2453269232188845966/master-0/data/info.pb
09:13:46.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.926720 27413 external_mini_cluster.cc:1428] Started /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu as pid 28498
09:13:46.927 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:13:46.927 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.197.125:37287
09:13:46.927 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
09:13:46.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.927868 27413 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu with pid 27505
09:13:46.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.928969 28587 sys_catalog.cc:263] Verifying existing consensus state
09:13:46.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.936925 28587 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: Bootstrap starting.
09:13:46.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:46.973933 28066 connection.cc:537] client connection to 127.26.197.125:37287 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:13:46.974 [ERROR - kudu-nio-0] (Connection.java:452) [peer master-127.26.197.125:37287(127.26.197.125:37287)] unexpected exception from downstream on [id: 0xf27176e9, L:/127.0.0.1:33554 - R:/127.26.197.125:37287]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) ~[?:1.8.0_275]
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:192)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379)
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255)
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
09:13:46.975 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state READY due to: [peer master-127.26.197.125:37287(127.26.197.125:37287)] unexpected exception from downstream on [id: 0xf27176e9, L:/127.0.0.1:33554 - R:/127.26.197.125:37287]
09:13:46.975 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.26.197.125:37287(127.26.197.125:37287) for tablet Kudu Master: [peer master-127.26.197.125:37287(127.26.197.125:37287)] unexpected exception from downstream on [id: 0xf27176e9, L:/127.0.0.1:33554 - R:/127.26.197.125:37287]
09:13:46.977 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.26.197.125:37287(127.26.197.125:37287)] handling channelInactive
09:13:46.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:46.979064 27769 connection.cc:537] client connection to 127.26.197.125:37287 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:13:46.987 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:13:46.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.990324 28191 heartbeater.cc:344] Connected to a master server at 127.26.197.126:42303
09:13:46.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:46.991335 27894 heartbeater.cc:344] Connected to a master server at 127.26.197.126:42303
09:13:46.992 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:46.993 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.126:42303(127.26.197.126:42303)] connecting to peer
09:13:46.993 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.26.197.126:42303(127.26.197.126:42303)] Successfully connected to peer
09:13:46.994 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:46.994 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:46.995 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:46.999 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.26.197.126
09:13:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.004866 28587 log.cc:826] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: Log is configured to *not* fsync() on all Append() calls
09:13:47.014 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x9c676750, L:/127.0.0.1:40488 - R:/127.26.197.126:42303] using SASL/PLAIN
09:13:47.016 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.126:42303: Catalog manager is not initialized. State: Starting
09:13:47.016 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.033 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.034 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.034 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.034 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.035 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.126:42303: Catalog manager is not initialized. State: Starting
09:13:47.036 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.045073 28587 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:13:47.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.045948 28587 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: Bootstrap complete.
09:13:47.053 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.056 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.126:42303: Catalog manager is not initialized. State: Starting
09:13:47.057 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.057 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.057 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.057 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.063971 28043 heartbeater.cc:344] Connected to a master server at 127.26.197.126:42303
09:13:47.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.069514 28587 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:47.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.071880 28587 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a70b898276748bba9d75a67edd84c5d, State: Initialized, Role: FOLLOWER
09:13:47.073 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.072638 28587 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:47.074 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.074 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.074 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.075582 28595 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.076932 28595 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: This master's current role is: FOLLOWER
09:13:47.077 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.126:42303: Catalog manager is not initialized. State: Starting
09:13:47.078 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.078709 28587 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: configured and running, proceeding with master startup.
09:13:47.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.109683 28606 catalog_manager.cc:1269] Loaded cluster ID: 812ace05a5054ff79e14cf188ae3b8f5
09:13:47.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.110195 28606 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: loading cluster ID for follower catalog manager: success
09:13:47.113 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.114 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.114 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.114 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.125479 28606 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: acquiring CA information for follower catalog manager: success
09:13:47.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.131628 28606 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:13:47.133 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.134 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.134 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.134 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.136 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.137 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] sending RPC to server master-127.26.197.125:37287, [43ms] received response from server master-127.26.197.125:37287: Network error: [peer master-127.26.197.125:37287(127.26.197.125:37287)] unexpected exception from downstream on [id: 0xf27176e9, L:/127.0.0.1:33554 - R:/127.26.197.125:37287], [43ms] delaying RPC due to: Network error: [peer master-127.26.197.125:37287(127.26.197.125:37287)] unexpected exception from downstream on [id: 0xf27176e9, L:/127.0.0.1:33554 - R:/127.26.197.125:37287], [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [62ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [64ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [84ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [102ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [103ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [104ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [124ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [125ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [125ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [142ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [145ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [146ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [146ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [182ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [184ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [187ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [187ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [202ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [204ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [204ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, deferred=Deferred@1548821878(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/331225083@ce29f43 -> (continuation of Deferred@2121304580 after retry RPC after error@1859517412) -> (continuation of Deferred@997857757 after retry RPC after error@603623798) -> (continuation of Deferred@1495277479 after retry RPC after error@418108860) -> (continuation of Deferred@1932825485 after retry RPC after error@78826940) -> (continuation of Deferred@1572218794 after retry RPC after error@464936853), errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/477708768@4cc90f95 -> (continuation of Deferred@2121304580 after retry RPC after error@1859517412) -> (continuation of Deferred@997857757 after retry RPC after error@603623798) -> (continuation of Deferred@1495277479 after retry RPC after error@418108860) -> (continuation of Deferred@1932825485 after retry RPC after error@78826940) -> (continuation of Deferred@1572218794 after retry RPC after error@464936853)))
09:13:47.138 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] sending RPC to server master-127.26.197.125:37287, [43ms] received response from server master-127.26.197.125:37287: Network error: [peer master-127.26.197.125:37287(127.26.197.125:37287)] unexpected exception from downstream on [id: 0xf27176e9, L:/127.0.0.1:33554 - R:/127.26.197.125:37287], [43ms] delaying RPC due to: Network error: [peer master-127.26.197.125:37287(127.26.197.125:37287)] unexpected exception from downstream on [id: 0xf27176e9, L:/127.0.0.1:33554 - R:/127.26.197.125:37287], [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [62ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [64ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [84ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [102ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [103ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [104ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [124ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [125ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [125ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [142ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [145ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [146ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [146ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [182ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [184ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [187ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [187ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [202ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [204ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [204ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [204ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, deferred=Deferred@1548821878(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$115/331225083@ce29f43 -> (continuation of Deferred@2121304580 after retry RPC after error@1859517412) -> (continuation of Deferred@997857757 after retry RPC after error@603623798) -> (continuation of Deferred@1495277479 after retry RPC after error@418108860) -> (continuation of Deferred@1932825485 after retry RPC after error@78826940) -> (continuation of Deferred@1572218794 after retry RPC after error@464936853), errback=org.apache.kudu.client.KuduTransaction$$Lambda$116/477708768@4cc90f95 -> (continuation of Deferred@2121304580 after retry RPC after error@1859517412) -> (continuation of Deferred@997857757 after retry RPC after error@603623798) -> (continuation of Deferred@1495277479 after retry RPC after error@418108860) -> (continuation of Deferred@1932825485 after retry RPC after error@78826940) -> (continuation of Deferred@1572218794 after retry RPC after error@464936853)))
09:13:47.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.125079 28498 master_runner.cc:186] Error getting master registration for 127.26.197.125:37287: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111),
09:13:47.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.148672 28498 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:13:47.153 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.155 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.155 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.156 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.157 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.173 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.174 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.174 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.174 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.177 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.177515 28609 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 80b3786ba7c94e0d96350abc26a3bee6)
09:13:47.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.178364 28609 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:47.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.182242 28609 leader_election.cc:290] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303), 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287)
09:13:47.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:47.186254 28364 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:47.194 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.195326 28552 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "3a70b898276748bba9d75a67edd84c5d" is_pre_election: true
09:13:47.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.196413 28552 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1cd9f73088cd4a8fa0cdc72a8dd908ba in term 2.
09:13:47.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.197860 28366 leader_election.cc:304] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1cd9f73088cd4a8fa0cdc72a8dd908ba, 3a70b898276748bba9d75a67edd84c5d; no voters:
09:13:47.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.198899 28609 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 3 FOLLOWER]: Leader pre-election won for term 4
09:13:47.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.199235 28609 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 3 FOLLOWER]: Starting leader election (detected failure of leader 80b3786ba7c94e0d96350abc26a3bee6)
09:13:47.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.199612 28609 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 3 FOLLOWER]: Advancing to term 4
09:13:47.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.207155 28609 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:47.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:47.209949 28364 leader_election.cc:336] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287): Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:47.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.212805 28552 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "3a70b898276748bba9d75a67edd84c5d"
09:13:47.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.213361 28552 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 2 FOLLOWER]: Advancing to term 4
09:13:47.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.212069 28609 leader_election.cc:290] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 4 election: Requested vote from peers 3a70b898276748bba9d75a67edd84c5d (127.26.197.126:42303), 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287)
09:13:47.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:47.217602 28364 leader_election.cc:336] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287): Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111)
09:13:47.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.219 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.223157 28552 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1cd9f73088cd4a8fa0cdc72a8dd908ba in term 4.
09:13:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.224431 28366 leader_election.cc:304] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1cd9f73088cd4a8fa0cdc72a8dd908ba, 3a70b898276748bba9d75a67edd84c5d; no voters: 80b3786ba7c94e0d96350abc26a3bee6
09:13:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.225308 28609 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 4 FOLLOWER]: Leader election won for term 4
09:13:47.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.227730 28609 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [term 4 LEADER]: Becoming Leader. State: Replica: 1cd9f73088cd4a8fa0cdc72a8dd908ba, State: Running, Role: LEADER
09:13:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.229096 28609 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 3.13, Last appended by leader: 13, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } }
09:13:47.232 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.234 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.234 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.234 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.245 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.245626 28613 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1cd9f73088cd4a8fa0cdc72a8dd908ba. Latest consensus state: current_term: 4 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.246536 28613 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: This master's current role is: LEADER
09:13:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.249150 28615 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:13:47.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.257244 28615 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a]
09:13:47.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.261461 28615 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=ff3716932b0f4b31bdb20d5031b99373]
09:13:47.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.268970 28615 tablet_loader.cc:96] loaded metadata for tablet 51549a3ef4aa4d7597f571ac38fa192f (table txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a])
09:13:47.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.270416 28615 tablet_loader.cc:96] loaded metadata for tablet 57f6e1962acd4ed59809df3980adf3c4 (table kudu_system.kudu_transactions [id=ff3716932b0f4b31bdb20d5031b99373])
09:13:47.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.271735 28615 tablet_loader.cc:96] loaded metadata for tablet 8b5bb375b623431b90ededae66a9070a (table txn_manager_fallback_rolling [id=03eaf326a2654e0dab58aa42e3c28b4a])
09:13:47.272 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.274 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.274 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.274 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.272845 28615 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:13:47.277 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303; will retry
09:13:47.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.278554 28615 catalog_manager.cc:1269] Loaded cluster ID: 812ace05a5054ff79e14cf188ae3b8f5
09:13:47.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.278810 28615 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:13:47.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.282223 28615 catalog_manager.cc:1514] Loading token signing keys...
09:13:47.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.285354 28615 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba: Loaded TSK: 0
09:13:47.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.286386 28615 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:13:47.313 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.26.197.125:37287(127.26.197.125:37287)] connecting to peer
09:13:47.314 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.314 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.197.125:37287(127.26.197.125:37287)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.314 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.197.125:37287: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287
09:13:47.317 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:13:47.317 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=6, TimeoutTracker(timeout=250, elapsed=164), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [3ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [4ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [4ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [5ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [22ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [25ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [59ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [59ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [66ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [67ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [82ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [86ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [93ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [122ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [125ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [125ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [162ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [163ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, deferred=Deferred@1651819785(state=PENDING, result=null, callback=(continuation of Deferred@1104976988 after retry RPC after error@541729025) -> (continuation of Deferred@917304034 after retry RPC after error@1916024243) -> (continuation of Deferred@1471117781 after retry RPC after error@647759880) -> (continuation of Deferred@2004272603 after retry RPC after error@607977259) -> (continuation of Deferred@150035368 after retry RPC after error@1315688182), errback=(continuation of Deferred@1104976988 after retry RPC after error@541729025) -> (continuation of Deferred@917304034 after retry RPC after error@1916024243) -> (continuation of Deferred@1471117781 after retry RPC after error@647759880) -> (continuation of Deferred@2004272603 after retry RPC after error@607977259) -> (continuation of Deferred@150035368 after retry RPC after error@1315688182))) after lookup
09:13:47.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.357913 28552 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [term 4 FOLLOWER]: Refusing update from remote peer 1cd9f73088cd4a8fa0cdc72a8dd908ba: Log matching property violated. Preceding OpId in replica: term: 2 index: 12. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
09:13:47.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:47.358063 28364 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba -> Peer 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287): Couldn't send request to peer 80b3786ba7c94e0d96350abc26a3bee6. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:13:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.360076 28613 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [LEADER]: Connected to new peer: Peer: permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 12, Time since last communication: 0.000s
09:13:47.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.396744 28595 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1cd9f73088cd4a8fa0cdc72a8dd908ba. Latest consensus state: current_term: 4 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:47.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.401774 28595 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: This master's current role is: FOLLOWER
09:13:47.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.398775 28623 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:47.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.402930 28623 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: This master's current role is: FOLLOWER
09:13:47.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.403592 28628 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:47.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:47.404255 28409 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:49176 (request call id 13) took 85 ms (client timeout 85 ms). Trace:
09:13:47.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.404580 28628 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a70b898276748bba9d75a67edd84c5d [sys.catalog]: This master's current role is: FOLLOWER
09:13:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:47.404698 28409 rpcz_store.cc:269] 1213 09:13:47.318523 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:13:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:47.318650 (+ 127us) service_pool.cc:225] Handling call
09:13:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 09:13:47.404050 (+ 85400us) inbound_call.cc:173] Queueing success response
09:13:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":572,"init.queue_time_us":1693,"thread_start_us":31927,"threads_started":5}
09:13:47.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.410681 28613 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a70b898276748bba9d75a67edd84c5d" member_type: VOTER last_known_addr { host: "127.26.197.126" port: 42303 } } peers { permanent_uuid: "80b3786ba7c94e0d96350abc26a3bee6" member_type: VOTER last_known_addr { host: "127.26.197.125" port: 37287 } } peers { permanent_uuid: "1cd9f73088cd4a8fa0cdc72a8dd908ba" member_type: VOTER last_known_addr { host: "127.26.197.124" port: 35671 } } }
09:13:47.413 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [3ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [4ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [4ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [5ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [22ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [25ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [59ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [59ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [66ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [67ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [82ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [86ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [93ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [122ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [125ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [125ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.26.197.125:37287,127.26.197.124:35671,127.26.197.126:42303) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.125:37287, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.124:35671, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.197.126:42303, [162ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.125:37287: Network error: Failed to connect to peer master-127.26.197.125:37287(127.26.197.125:37287): Connection refused: /127.26.197.125:37287, [163ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.124:35671: OK, [166ms] sending RPC to server master-127.26.197.124:35671, [166ms] Sub RPC ConnectToMaster: received response from server master-127.26.197.126:42303: OK, deferred=Deferred@1651819785(state=PENDING, result=null, callback=(continuation of Deferred@1104976988 after retry RPC after error@541729025) -> (continuation of Deferred@917304034 after retry RPC after error@1916024243) -> (continuation of Deferred@1471117781 after retry RPC after error@647759880) -> (continuation of Deferred@2004272603 after retry RPC after error@607977259) -> (continuation of Deferred@150035368 after retry RPC after error@1315688182) -> (continuation of Deferred@757859510 after retry RPC@1022783042), errback=(continuation of Deferred@1104976988 after retry RPC after error@541729025) -> (continuation of Deferred@917304034 after retry RPC after error@1916024243) -> (continuation of Deferred@1471117781 after retry RPC after error@647759880) -> (continuation of Deferred@2004272603 after retry RPC after error@607977259) -> (continuation of Deferred@150035368 after retry RPC after error@1315688182) -> (continuation of Deferred@757859510 after retry RPC@1022783042)))
09:13:47.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.411928 28613 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba [sys.catalog]: This master's current role is: LEADER
09:13:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.437351 28379 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:49194:
09:13:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:13:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:13:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:13:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:13:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:13:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:13:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:13:47.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:47.442198 28379 catalog_manager.cc:7016] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:13:47.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.511538 28073 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.027s from last keepalive heartbeat (effective timeout is 1.000s)
09:13:47.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:47.545221 28633 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
09:13:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:47.559371 28634 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
09:13:47.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:47.819672 28364 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba -> Peer 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287): Couldn't send request to peer 80b3786ba7c94e0d96350abc26a3bee6. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:13:47.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.982710 28541 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a8d35d68bd544dd1bb30304d9ec69d5c" instance_seqno: 1765617217305234) as {username='slave'} at 127.26.197.67:45827; Asking this server to re-register.
09:13:47.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.983950 28190 heartbeater.cc:499] Master 127.26.197.124:35671 was elected leader, sending a full tablet report...
09:13:47.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.984658 28191 heartbeater.cc:461] Registering TS with master...
09:13:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:47.988693 28541 ts_manager.cc:194] Registered new tserver with Master: a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67:45421)
09:13:48.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:48.000918 28541 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b03cd02817ea44cfa620f4c79f743171" instance_seqno: 1765617212783843) as {username='slave'} at 127.26.197.65:37713; Asking this server to re-register.
09:13:48.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:48.002173 27894 heartbeater.cc:461] Registering TS with master...
09:13:48.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:48.005252 28541 ts_manager.cc:194] Registered new tserver with Master: b03cd02817ea44cfa620f4c79f743171 (127.26.197.65:45603)
09:13:48.019 [DEBUG - kudu-nio-1] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: a8d35d68bd544dd1bb30304d9ec69d5c (127.26.197.67:45421): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1765617218 last_transition_timestamp: 1765617227
09:13:48.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:48.055491 28340 master_runner.cc:186] Error getting master registration for 127.26.197.125:37287: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111),
09:13:48.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:48.055958 28340 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:13:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:48.070019 28541 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a968a863d78b44118abd134c71c37377" instance_seqno: 1765617215062400) as {username='slave'} at 127.26.197.66:51013; Asking this server to re-register.
09:13:48.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:48.071329 28043 heartbeater.cc:461] Registering TS with master...
09:13:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:48.074709 28541 ts_manager.cc:194] Registered new tserver with Master: a968a863d78b44118abd134c71c37377 (127.26.197.66:43885)
09:13:48.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:48.374133 28364 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba -> Peer 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287): Couldn't send request to peer 80b3786ba7c94e0d96350abc26a3bee6. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:13:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:48.809399 28364 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1cd9f73088cd4a8fa0cdc72a8dd908ba -> Peer 80b3786ba7c94e0d96350abc26a3bee6 (127.26.197.125:37287): Couldn't send request to peer 80b3786ba7c94e0d96350abc26a3bee6. Status: Network error: Client connection negotiation failed: client connection to 127.26.197.125:37287: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
09:13:48.996 [DEBUG - main] (Connection.java:575) [peer a968a863d78b44118abd134c71c37377(127.26.197.66:43885)] disconnecting while in state READY
09:13:48.996 [DEBUG - main] (Connection.java:575) [peer master-127.26.197.126:42303(127.26.197.126:42303)] disconnecting while in state READY
09:13:48.996 [DEBUG - main] (Connection.java:575) [peer master-127.26.197.124:35671(127.26.197.124:35671)] disconnecting while in state READY
09:13:48.996 [DEBUG - main] (Connection.java:575) [peer b03cd02817ea44cfa620f4c79f743171(127.26.197.65:45603)] disconnecting while in state READY
09:13:48.996 [DEBUG - main] (Connection.java:575) [peer master-127.26.197.125:37287(127.26.197.125:37287)] disconnecting while in state TERMINATED
09:13:48.998 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:13:48.999 [DEBUG - kudu-nio-1] (Connection.java:239) [peer a968a863d78b44118abd134c71c37377(127.26.197.66:43885)] handling channelInactive
09:13:48.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer a968a863d78b44118abd134c71c37377(127.26.197.66:43885)] cleaning up while in state READY due to: connection closed
09:13:48.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:48.999198 27413 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu with pid 27727
09:13:48.999 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.26.197.126:42303(127.26.197.126:42303)] handling channelInactive
09:13:49.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.197.126:42303(127.26.197.126:42303)] cleaning up while in state READY due to: connection closed
09:13:49.000 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.26.197.124:35671(127.26.197.124:35671)] handling channelInactive
09:13:49.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.197.124:35671(127.26.197.124:35671)] cleaning up while in state READY due to: connection closed
09:13:49.001 [DEBUG - kudu-nio-1] (Connection.java:239) [peer b03cd02817ea44cfa620f4c79f743171(127.26.197.65:45603)] handling channelInactive
09:13:49.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer b03cd02817ea44cfa620f4c79f743171(127.26.197.65:45603)] cleaning up while in state READY due to: connection closed
09:13:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:49.026503 27413 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu with pid 27897
09:13:49.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:49.055558 27413 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu with pid 28046
09:13:49.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:49.085593 27413 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu with pid 28498
09:13:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 09:13:49.116062 28366 connection.cc:537] client connection to 127.26.197.126:42303 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:13:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 09:13:49.116642 27413 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task0xFmrh/build/tsan/bin/kudu with pid 28340
09:13:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T09:13:50Z chronyd exiting
09:13:51.262 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)