02:10:59.355 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:10:59.355 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task5Jg9dI/test-tmp
02:10:59.355 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:10:59.356 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task5Jg9dI/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:10:59.358 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:10:59.697 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:10:59.697 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:10:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:10:59.700590 24158 mini_oidc.cc:150] Starting JWKS server
02:10:59.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:10:59.703886 24158 webserver.cc:533] Webserver started at http://127.0.0.1:39597/ using document root <none> and password file <none>
02:10:59.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:10:59.704295 24158 mini_oidc.cc:186] Starting OIDC Discovery server
02:10:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:10:59.706064 24158 webserver.cc:533] Webserver started at http://127.0.0.1:35613/ using document root <none> and password file <none>
02:10:59.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T02:10:59Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:10:59.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T02:10:59Z Disabled control of system clock
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:10:59.776249 24158 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/wal
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/data
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/logs
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/data/info.pb
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:10:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.190:42715
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.190
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:10:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39597/jwks.json with env {}
02:11:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.106371 24171 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:00.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.107029 24171 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:00.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.107728 24171 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.141274 24171 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:11:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.141569 24171 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.141782 24171 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.142017 24171 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:11:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.142230 24171 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.142390 24171 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39597/jwks.json
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:00.179703 24171 master_runner.cc:387] Master server non-default flags:
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/logs
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/data
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/wal
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.190:42715
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39597/jwks.json
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/data/info.pb
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.190
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:00.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:00.181118 24171 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:00.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:00.182767 24171 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.194072 24177 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:01.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:00.194450 24178 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:01.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:01.471978 24171 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.278s user 0.394s sys 0.872s
02:11:01.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:01.472379 24171 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.278s user 0.394s sys 0.872s
02:11:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:01.472983 24180 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.473030 24171 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:01.472250 24179 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1275 milliseconds
02:11:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.475204 24171 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:01.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.478725 24171 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:01.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.480386 24171 hybrid_clock.cc:648] HybridClock initialized: now 1786327861480330 us; error 47 us; skew 500 ppm
02:11:01.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.481686 24171 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.491389 24171 webserver.cc:533] Webserver started at http://127.23.151.190:44561/ using document root <none> and password file <none>
02:11:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.492964 24171 fs_manager.cc:362] Metadata directory not provided
02:11:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.493301 24171 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:01.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.493960 24171 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:11:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.501346 24171 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/data/instance:
02:11:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e08b19cf21d49b6a63ebd9b63863591"
02:11:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:01 on dist-test-slave-rnrw"
02:11:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.503187 24171 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/wal/instance:
02:11:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e08b19cf21d49b6a63ebd9b63863591"
02:11:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:01 on dist-test-slave-rnrw"
02:11:01.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.514402 24171 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.010s sys 0.003s
02:11:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.523181 24187 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.524602 24171 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.524952 24171 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/wal
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e08b19cf21d49b6a63ebd9b63863591"
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:01 on dist-test-slave-rnrw"
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.525283 24171 fs_report.cc:389] FS layout report
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/wal
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/wal
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/data/data
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.612749 24171 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.614262 24171 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.614686 24171 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:01.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.690143 24171 rpc_server.cc:307] RPC server started. Bound to: 127.23.151.190:42715
02:11:01.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.690254 24243 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.151.190:42715 every 8 connection(s)
02:11:01.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.692902 24171 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/data/info.pb
02:11:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.694809 24158 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 24171
02:11:01.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.695382 24158 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-0/wal/instance
02:11:01.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.700338 24244 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.701284 24158 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/wal
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/data
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/logs
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/data/info.pb
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.189:40229
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.189
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39597/jwks.json with env {}
02:11:01.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.714622 24244 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } has no permanent_uuid. Determining permanent_uuid...
02:11:01.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.749941 24244 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } has no permanent_uuid. Determining permanent_uuid...
02:11:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:01.752638 24191 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.151.189:40229: connect: Connection refused (error 111)
02:11:01.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:01.757344 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.189:40229: Network error: Client connection negotiation failed: client connection to 127.23.151.189:40229: connect: Connection refused (error 111)
02:11:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.806926 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } attempt: 1
02:11:01.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:01.811545 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.189:40229: Network error: Client connection negotiation failed: client connection to 127.23.151.189:40229: connect: Connection refused (error 111)
02:11:01.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.879951 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } attempt: 2
02:11:01.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:01.884562 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.189:40229: Network error: Client connection negotiation failed: client connection to 127.23.151.189:40229: connect: Connection refused (error 111)
02:11:01.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:01.976069 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } attempt: 3
02:11:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:01.980603 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.189:40229: Network error: Client connection negotiation failed: client connection to 127.23.151.189:40229: connect: Connection refused (error 111)
02:11:02.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.031682 24246 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:02.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.032248 24246 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:02.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.032861 24246 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.066064 24246 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:11:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.066377 24246 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.066570 24246 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.066779 24246 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:11:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.066973 24246 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.067121 24246 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39597/jwks.json
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:02.105271 24246 master_runner.cc:387] Master server non-default flags:
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/logs
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/data
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/wal
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.189:40229
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39597/jwks.json
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/data/info.pb
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.189
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:02.106585 24246 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:02.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:02.108202 24246 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.119336 24257 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:02.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:02.124084 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } attempt: 4
02:11:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.131242 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.189:40229: Network error: Client connection negotiation failed: client connection to 127.23.151.189:40229: connect: Connection refused (error 111)
02:11:02.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:02.430744 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } attempt: 5
02:11:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.441077 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.189:40229: Network error: Client connection negotiation failed: client connection to 127.23.151.189:40229: connect: Connection refused (error 111)
02:11:02.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:02.988780 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } attempt: 6
02:11:03.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:03.014292 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.189:40229: Network error: Client connection negotiation failed: client connection to 127.23.151.189:40229: connect: Connection refused (error 111)
02:11:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:03.525327 24256 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 24246
02:11:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:03.682607 24256 kernel_stack_watchdog.cc:198] Thread 24246 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
02:11:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:11:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:11:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:11:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:11:03.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:03.684199 24246 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.564s user 0.588s sys 0.974s
02:11:03.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:03.684733 24246 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.565s user 0.588s sys 0.974s
02:11:03.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:02.120249 24258 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:03.686213 24259 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1563 milliseconds
02:11:03.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.686815 24246 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:03.686868 24260 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.691370 24246 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.694198 24246 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:03.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.695598 24246 hybrid_clock.cc:648] HybridClock initialized: now 1786327863695554 us; error 37 us; skew 500 ppm
02:11:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.696468 24246 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.703065 24246 webserver.cc:533] Webserver started at http://127.23.151.189:40947/ using document root <none> and password file <none>
02:11:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.704061 24246 fs_manager.cc:362] Metadata directory not provided
02:11:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.704320 24246 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:03.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.704871 24246 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:11:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.709316 24246 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/data/instance:
02:11:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e736a559da9440ec9278ca3d6b3280d8"
02:11:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:03 on dist-test-slave-rnrw"
02:11:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.710637 24246 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/wal/instance:
02:11:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e736a559da9440ec9278ca3d6b3280d8"
02:11:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:03 on dist-test-slave-rnrw"
02:11:03.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.718333 24246 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
02:11:03.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.724140 24269 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.725292 24246 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:11:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.725658 24246 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/wal
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e736a559da9440ec9278ca3d6b3280d8"
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:03 on dist-test-slave-rnrw"
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.726007 24246 fs_report.cc:389] FS layout report
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/wal
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/wal
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/data/data
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:03.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.778882 24246 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:03.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.780458 24246 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:03.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.781062 24246 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:03.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.855508 24246 rpc_server.cc:307] RPC server started. Bound to: 127.23.151.189:40229
02:11:03.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.855608 24324 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.151.189:40229 every 8 connection(s)
02:11:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.858353 24246 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/data/info.pb
02:11:03.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.862083 24158 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 24246
02:11:03.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.862635 24158 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-1/wal/instance
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.865711 24158 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/wal
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/data
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/logs
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/data/info.pb
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.188:46563
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.188
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39597/jwks.json with env {}
02:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.866050 24325 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:11:03.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.879408 24325 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } has no permanent_uuid. Determining permanent_uuid...
02:11:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.902565 24325 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } has no permanent_uuid. Determining permanent_uuid...
02:11:03.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.917562 24325 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } has no permanent_uuid. Determining permanent_uuid...
02:11:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:03.920094 24270 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:03.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:03.924300 24325 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:03.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:03.973814 24325 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 1
02:11:03.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:03.978204 24325 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:04.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.046651 24325 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 2
02:11:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.050977 24325 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:04.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.074924 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } attempt: 7
02:11:04.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.087406 24244 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } has no permanent_uuid. Determining permanent_uuid...
02:11:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.093935 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:04.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.142356 24325 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 3
02:11:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.146796 24325 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:04.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.152427 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 1
02:11:04.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.156945 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.203369 24327 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:04.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.204068 24327 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:04.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.204694 24327 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.238374 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 2
02:11:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.238634 24327 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:11:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.239085 24327 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.239418 24327 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.239790 24327 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:11:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.240104 24327 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.240357 24327 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39597/jwks.json
02:11:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.242803 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:04.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.279497 24327 master_runner.cc:387] Master server non-default flags:
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/logs
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/data
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/wal
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.188:46563
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39597/jwks.json
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/data/info.pb
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.188
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:04.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:04.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.280987 24327 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:04.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.282642 24327 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.290236 24325 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 4
02:11:04.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.294046 24340 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:04.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.297578 24325 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.328306 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 3
02:11:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.332739 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:04.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.473237 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 4
02:11:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.480688 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.597134 24325 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 5
02:11:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.604588 24325 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:04.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:04.764321 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 5
02:11:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.771647 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:05.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.152271 24325 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 6
02:11:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:05.163807 24325 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:05.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.324234 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 6
02:11:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:05.333768 24244 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.151.188:46563: Network error: Client connection negotiation failed: client connection to 127.23.151.188:46563: connect: Connection refused (error 111)
02:11:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.297710 24344 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:05.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:04.297013 24342 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:05.561532 24343 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1261 milliseconds
02:11:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.561651 24327 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.562958 24327 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:05.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.565714 24327 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.567076 24327 hybrid_clock.cc:648] HybridClock initialized: now 1786327865567043 us; error 51 us; skew 500 ppm
02:11:05.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.567898 24327 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.575040 24327 webserver.cc:533] Webserver started at http://127.23.151.188:36565/ using document root <none> and password file <none>
02:11:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.576009 24327 fs_manager.cc:362] Metadata directory not provided
02:11:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.576205 24327 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.576715 24327 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:11:05.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.581681 24327 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/data/instance:
02:11:05.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cee0500569e844fca611d9a80c9baad1"
02:11:05.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:05 on dist-test-slave-rnrw"
02:11:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.582787 24327 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/wal/instance:
02:11:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cee0500569e844fca611d9a80c9baad1"
02:11:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:05 on dist-test-slave-rnrw"
02:11:05.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.590539 24327 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
02:11:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.596791 24356 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.598029 24327 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
02:11:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.598328 24327 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/wal
02:11:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cee0500569e844fca611d9a80c9baad1"
02:11:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:05 on dist-test-slave-rnrw"
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.598649 24327 fs_report.cc:389] FS layout report
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/wal
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/wal
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/data/data
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:05.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.649977 24327 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.651463 24327 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.651875 24327 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.729961 24327 rpc_server.cc:307] RPC server started. Bound to: 127.23.151.188:46563
02:11:05.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.730062 24412 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.151.188:46563 every 8 connection(s)
02:11:05.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.732749 24327 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/data/info.pb
02:11:05.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.738883 24158 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 24327
02:11:05.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.739391 24158 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/master-2/wal/instance
02:11:05.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.739428 24413 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:11:05.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.756793 24413 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } has no permanent_uuid. Determining permanent_uuid...
02:11:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.784263 24413 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } has no permanent_uuid. Determining permanent_uuid...
02:11:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.795291 24413 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } has no permanent_uuid. Determining permanent_uuid...
02:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.811204 24413 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1: Bootstrap starting.
02:11:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.817610 24413 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1: Neither blocks nor log segments found. Creating new log.
02:11:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.819766 24413 log.cc:826] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1: Log is configured to *not* fsync() on all Append() calls
02:11:05.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.824384 24413 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1: No bootstrap required, opened a new log
02:11:05.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.844179 24413 raft_consensus.cc:359] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } }
02:11:05.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.844936 24413 raft_consensus.cc:385] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:11:05.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.845242 24413 raft_consensus.cc:740] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cee0500569e844fca611d9a80c9baad1, State: Initialized, Role: FOLLOWER
02:11:05.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.846086 24413 consensus_queue.cc:260] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } }
02:11:05.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.848596 24423 sys_catalog.cc:455] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } } }
02:11:05.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.849251 24423 sys_catalog.cc:458] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [sys.catalog]: This master's current role is: FOLLOWER
02:11:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.850103 24413 sys_catalog.cc:565] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [sys.catalog]: configured and running, proceeding with master startup.
02:11:05.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:05.888923 24436 catalog_manager.cc:1594] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:05.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:05.889272 24436 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:05.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.876499 24327 master_runner.cc:187] Error getting master registration for 127.23.151.190:42715: OK, instance_id { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" instance_seqno: 1786327861668685 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:11:05.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:05.906209 24327 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:11:05.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:05.907195 24362 tablet.cc:2406] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:11:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.224386 24325 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 7
02:11:06.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.244331 24325 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8: Bootstrap starting.
02:11:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.250025 24325 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8: Neither blocks nor log segments found. Creating new log.
02:11:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.251911 24325 log.cc:826] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8: Log is configured to *not* fsync() on all Append() calls
02:11:06.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.256477 24325 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8: No bootstrap required, opened a new log
02:11:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.276465 24325 raft_consensus.cc:359] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } }
02:11:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.277143 24325 raft_consensus.cc:385] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:11:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.277359 24325 raft_consensus.cc:740] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e736a559da9440ec9278ca3d6b3280d8, State: Initialized, Role: FOLLOWER
02:11:06.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.278034 24325 consensus_queue.cc:260] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } }
02:11:06.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.280730 24440 sys_catalog.cc:455] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } } }
02:11:06.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.281579 24440 sys_catalog.cc:458] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [sys.catalog]: This master's current role is: FOLLOWER
02:11:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.282748 24325 sys_catalog.cc:565] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [sys.catalog]: configured and running, proceeding with master startup.
02:11:06.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:06.289048 24275 tablet.cc:2406] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:11:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:06.315156 24452 catalog_manager.cc:1594] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:06.315766 24452 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:06.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.310060 24246 master_runner.cc:187] Error getting master registration for 127.23.151.190:42715: OK, instance_id { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" instance_seqno: 1786327861668685 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:11:06.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.341655 24246 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:11:06.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.344018 24423 raft_consensus.cc:493] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.344830 24423 raft_consensus.cc:515] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } }
02:11:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.352152 24298 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cee0500569e844fca611d9a80c9baad1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e736a559da9440ec9278ca3d6b3280d8" is_pre_election: true
02:11:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.353102 24298 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cee0500569e844fca611d9a80c9baad1 in term 0.
02:11:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.354945 24360 leader_election.cc:304] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cee0500569e844fca611d9a80c9baad1, e736a559da9440ec9278ca3d6b3280d8; no voters:
02:11:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.356707 24423 leader_election.cc:290] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e08b19cf21d49b6a63ebd9b63863591 (127.23.151.190:42715), e736a559da9440ec9278ca3d6b3280d8 (127.23.151.189:40229)
02:11:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.357234 24423 raft_consensus.cc:2804] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:11:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.357573 24423 raft_consensus.cc:493] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:11:06.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.357882 24423 raft_consensus.cc:3060] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 0 FOLLOWER]: Advancing to term 1
02:11:06.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.367231 24244 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } attempt: 7
02:11:06.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.366709 24423 raft_consensus.cc:515] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } }
02:11:06.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.380841 24298 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cee0500569e844fca611d9a80c9baad1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e736a559da9440ec9278ca3d6b3280d8"
02:11:06.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.381465 24298 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [term 0 FOLLOWER]: Advancing to term 1
02:11:06.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.388492 24298 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cee0500569e844fca611d9a80c9baad1 in term 1.
02:11:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.391875 24360 leader_election.cc:304] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cee0500569e844fca611d9a80c9baad1, e736a559da9440ec9278ca3d6b3280d8; no voters:
02:11:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.393003 24454 raft_consensus.cc:2804] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 1 FOLLOWER]: Leader election won for term 1
02:11:06.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.393486 24423 leader_election.cc:290] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [CANDIDATE]: Term 1 election: Requested vote from peers 9e08b19cf21d49b6a63ebd9b63863591 (127.23.151.190:42715), e736a559da9440ec9278ca3d6b3280d8 (127.23.151.189:40229)
02:11:06.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.398845 24454 raft_consensus.cc:697] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [term 1 LEADER]: Becoming Leader. State: Replica: cee0500569e844fca611d9a80c9baad1, State: Running, Role: LEADER
02:11:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.399976 24454 consensus_queue.cc:237] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } }
02:11:06.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.414505 24423 sys_catalog.cc:455] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cee0500569e844fca611d9a80c9baad1. Latest consensus state: current_term: 1 leader_uuid: "cee0500569e844fca611d9a80c9baad1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } } }
02:11:06.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.415230 24423 sys_catalog.cc:458] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [sys.catalog]: This master's current role is: LEADER
02:11:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.425737 24458 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
02:11:06.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.428025 24244 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591: Bootstrap starting.
02:11:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.435573 24458 catalog_manager.cc:1520] Initializing Kudu cluster ID...
02:11:06.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.445171 24244 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591: Neither blocks nor log segments found. Creating new log.
02:11:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.450187 24244 log.cc:826] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591: Log is configured to *not* fsync() on all Append() calls
02:11:06.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.466849 24244 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591: No bootstrap required, opened a new log
02:11:06.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.486424 24298 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [term 1 FOLLOWER]: Refusing update from remote peer cee0500569e844fca611d9a80c9baad1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:11:06.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.497514 24454 consensus_queue.cc:1048] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:11:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.501647 24244 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } }
02:11:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.502533 24244 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:11:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.502849 24244 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e08b19cf21d49b6a63ebd9b63863591, State: Initialized, Role: FOLLOWER
02:11:06.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.504120 24244 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } }
02:11:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.520354 24244 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [sys.catalog]: configured and running, proceeding with master startup.
02:11:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.520973 24215 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [term 0 FOLLOWER]: Advancing to term 1
02:11:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.522750 24461 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } } }
02:11:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.523651 24461 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [sys.catalog]: This master's current role is: FOLLOWER
02:11:06.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.525112 24217 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cee0500569e844fca611d9a80c9baad1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e08b19cf21d49b6a63ebd9b63863591" is_pre_election: true
02:11:06.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.528067 24216 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cee0500569e844fca611d9a80c9baad1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e08b19cf21d49b6a63ebd9b63863591"
02:11:06.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.534338 24215 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [term 1 FOLLOWER]: Refusing update from remote peer cee0500569e844fca611d9a80c9baad1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:11:06.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.537317 24454 consensus_queue.cc:1048] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:11:06.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.581598 24461 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cee0500569e844fca611d9a80c9baad1. Latest consensus state: current_term: 1 leader_uuid: "cee0500569e844fca611d9a80c9baad1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } } }
02:11:06.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.582886 24461 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [sys.catalog]: This master's current role is: FOLLOWER
02:11:06.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.595881 24459 mvcc.cc:204] Tried to move back new op lower bound from 7316798941101350912 to 7316798940786765824. Current Snapshot: MvccSnapshot[applied={T|T < 7316798941101350912}]
02:11:06.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.598445 24440 sys_catalog.cc:455] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cee0500569e844fca611d9a80c9baad1. Latest consensus state: current_term: 1 leader_uuid: "cee0500569e844fca611d9a80c9baad1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } } }
02:11:06.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.599179 24440 sys_catalog.cc:458] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [sys.catalog]: This master's current role is: FOLLOWER
02:11:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.604806 24468 mvcc.cc:204] Tried to move back new op lower bound from 7316798941101350912 to 7316798940786765824. Current Snapshot: MvccSnapshot[applied={T|T < 7316798941101350912}]
02:11:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.607669 24464 mvcc.cc:204] Tried to move back new op lower bound from 7316798941101350912 to 7316798940786765824. Current Snapshot: MvccSnapshot[applied={T|T < 7316798941101350912}]
02:11:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.608569 24440 sys_catalog.cc:455] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cee0500569e844fca611d9a80c9baad1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } } }
02:11:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.609237 24440 sys_catalog.cc:458] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8 [sys.catalog]: This master's current role is: FOLLOWER
02:11:06.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.632277 24456 sys_catalog.cc:455] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cee0500569e844fca611d9a80c9baad1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } } }
02:11:06.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.633176 24456 sys_catalog.cc:458] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [sys.catalog]: This master's current role is: LEADER
02:11:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.633632 24461 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cee0500569e844fca611d9a80c9baad1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } } }
02:11:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.633755 24454 sys_catalog.cc:455] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cee0500569e844fca611d9a80c9baad1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e08b19cf21d49b6a63ebd9b63863591" member_type: VOTER last_known_addr { host: "127.23.151.190" port: 42715 } } peers { permanent_uuid: "e736a559da9440ec9278ca3d6b3280d8" member_type: VOTER last_known_addr { host: "127.23.151.189" port: 40229 } } peers { permanent_uuid: "cee0500569e844fca611d9a80c9baad1" member_type: VOTER last_known_addr { host: "127.23.151.188" port: 46563 } } }
02:11:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.634423 24461 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591 [sys.catalog]: This master's current role is: FOLLOWER
02:11:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.634517 24454 sys_catalog.cc:458] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1 [sys.catalog]: This master's current role is: LEADER
02:11:06.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.640884 24458 catalog_manager.cc:1383] Generated new cluster ID: 93f6c5579c8d46be808a60cd0db412f9
02:11:06.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.641167 24458 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
02:11:06.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.693629 24482 catalog_manager.cc:1295] Loaded cluster ID: 93f6c5579c8d46be808a60cd0db412f9
02:11:06.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.695726 24482 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591: loading cluster ID for follower catalog manager: success
02:11:06.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:06.699929 24482 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591: acquiring CA information for follower catalog manager: Not found: root CA entry not found
02:11:06.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:06.700232 24482 catalog_manager.cc:883] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
02:11:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.708778 24458 catalog_manager.cc:1406] Generated new certificate authority record
02:11:06.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.710839 24458 catalog_manager.cc:1540] Loading token signing keys...
02:11:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.755980 24458 catalog_manager.cc:6092] T 00000000000000000000000000000000 P cee0500569e844fca611d9a80c9baad1: Generated new TSK 0
02:11:06.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.757380 24458 catalog_manager.cc:1550] Initializing in-progress tserver states...
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:06.812568 24158 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/wal
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/data
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/logs
02:11:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/data/info.pb
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.129:0
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.151.129
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.129
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:11:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:07.322314 24452 catalog_manager.cc:1295] Loaded cluster ID: 93f6c5579c8d46be808a60cd0db412f9
02:11:07.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:07.322873 24452 catalog_manager.cc:1588] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8: loading cluster ID for follower catalog manager: success
02:11:07.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:07.336910 24452 catalog_manager.cc:1610] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8: acquiring CA information for follower catalog manager: success
02:11:07.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:07.351822 24452 catalog_manager.cc:1638] T 00000000000000000000000000000000 P e736a559da9440ec9278ca3d6b3280d8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:11:07.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:07.407895 24484 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:07.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:07.408617 24484 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:07.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:07.409417 24484 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:07.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:07.472499 24484 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:07.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:07.473932 24484 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.151.129
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:07.542498 24484 tablet_server_runner.cc:78] Tablet server non-default flags:
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/logs
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/data
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/wal
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.129:0
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/data/info.pb
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.129
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.151.129
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:07.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:07.544304 24484 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:07.546742 24484 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:07.584522 24490 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:07.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:07.718942 24482 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591: acquiring CA information for follower catalog manager: success
02:11:07.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:07.724480 24482 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 9e08b19cf21d49b6a63ebd9b63863591: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:11:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:08.970559 24489 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 24484
02:11:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:09.412164 24484 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.823s user 0.619s sys 1.149s
02:11:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:07.599565 24491 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:09.420372 24484 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.831s user 0.619s sys 1.149s
02:11:09.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.434791 24484 server_base.cc:1061] running on GCE node
02:11:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:09.436821 24495 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:09.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.438550 24484 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:09.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.453085 24484 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:09.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.456949 24484 hybrid_clock.cc:648] HybridClock initialized: now 1786327869456859 us; error 91 us; skew 500 ppm
02:11:09.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.458324 24484 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.491978 24484 webserver.cc:533] Webserver started at http://127.23.151.129:33603/ using document root <none> and password file <none>
02:11:09.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.493880 24484 fs_manager.cc:362] Metadata directory not provided
02:11:09.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.494490 24484 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:09.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.495293 24484 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:11:09.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.502710 24484 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/data/instance:
02:11:09.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5b76fbb9e2f4cb09a8bbd09b9c66382"
02:11:09.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:09 on dist-test-slave-rnrw"
02:11:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.504858 24484 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/wal/instance:
02:11:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5b76fbb9e2f4cb09a8bbd09b9c66382"
02:11:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:09 on dist-test-slave-rnrw"
02:11:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.527340 24484 fs_manager.cc:696] Time spent creating directory manager: real 0.021s user 0.011s sys 0.008s
02:11:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.541472 24500 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.543166 24484 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.004s sys 0.004s
02:11:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.543634 24484 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/wal
02:11:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5b76fbb9e2f4cb09a8bbd09b9c66382"
02:11:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:09 on dist-test-slave-rnrw"
02:11:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.544108 24484 fs_report.cc:389] FS layout report
02:11:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/wal
02:11:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/wal
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/data/data
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.622185 24484 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.624294 24484 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:09.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.624915 24484 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.628657 24484 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:11:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.634895 24484 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:11:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.635178 24484 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:11:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.635512 24484 ts_tablet_manager.cc:616] Registered 0 tablets
02:11:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.635756 24484 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:11:09.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.926734 24484 rpc_server.cc:307] RPC server started. Bound to: 127.23.151.129:32873
02:11:09.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.926882 24627 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.151.129:32873 every 8 connection(s)
02:11:09.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.930631 24484 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/data/info.pb
02:11:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.931835 24158 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 24484
02:11:09.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.932578 24158 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-0/wal/instance
02:11:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:09.991417 24158 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/wal
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/data
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/logs
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/data/info.pb
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.130:0
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.151.130
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.130
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:11:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.005427 24629 heartbeater.cc:344] Connected to a master server at 127.23.151.188:46563
02:11:10.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.006003 24629 heartbeater.cc:461] Registering TS with master...
02:11:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.007395 24629 heartbeater.cc:507] Master 127.23.151.188:46563 requested a full tablet report, sending...
02:11:10.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.011032 24375 ts_manager.cc:194] Registered new tserver with Master: b5b76fbb9e2f4cb09a8bbd09b9c66382 (127.23.151.129:32873)
02:11:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.014066 24375 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.151.129:36307
02:11:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.025540 24628 heartbeater.cc:344] Connected to a master server at 127.23.151.189:40229
02:11:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.025902 24628 heartbeater.cc:461] Registering TS with master...
02:11:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.026467 24628 heartbeater.cc:507] Master 127.23.151.189:40229 requested a full tablet report, sending...
02:11:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.027302 24634 heartbeater.cc:344] Connected to a master server at 127.23.151.190:42715
02:11:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.027664 24634 heartbeater.cc:461] Registering TS with master...
02:11:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.028277 24634 heartbeater.cc:507] Master 127.23.151.190:42715 requested a full tablet report, sending...
02:11:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.029848 24287 ts_manager.cc:194] Registered new tserver with Master: b5b76fbb9e2f4cb09a8bbd09b9c66382 (127.23.151.129:32873)
02:11:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.032997 24207 ts_manager.cc:194] Registered new tserver with Master: b5b76fbb9e2f4cb09a8bbd09b9c66382 (127.23.151.129:32873)
02:11:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:10.410115 24638 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:10.410615 24638 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:10.411026 24638 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:10.445017 24638 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:10.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:10.445807 24638 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.151.130
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.483795 24638 tablet_server_runner.cc:78] Tablet server non-default flags:
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/logs
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/data
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/wal
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.130:0
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/data/info.pb
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.130
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.151.130
02:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:10.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:11:10.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:10.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:10.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:10.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:10.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:10.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:10.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.485298 24638 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:10.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:10.486939 24638 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:10.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:10.501340 24645 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:11.018843 24629 heartbeater.cc:499] Master 127.23.151.188:46563 was elected leader, sending a full tablet report...
02:11:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:10.502377 24646 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:11.902664 24644 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 24638
02:11:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.262248 24644 kernel_stack_watchdog.cc:198] Thread 24638 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
02:11:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:11:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:11:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:11:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:11:12.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.263266 24638 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.760s user 0.000s sys 0.001s
02:11:12.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.263764 24638 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.761s user 0.001s sys 0.001s
02:11:12.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.264474 24649 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.268913 24647 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1765 milliseconds
02:11:12.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.268920 24638 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:12.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.270308 24638 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.272476 24638 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:12.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.273926 24638 hybrid_clock.cc:648] HybridClock initialized: now 1786327872273884 us; error 41 us; skew 500 ppm
02:11:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.274914 24638 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:12.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.281795 24638 webserver.cc:533] Webserver started at http://127.23.151.130:38553/ using document root <none> and password file <none>
02:11:12.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.282696 24638 fs_manager.cc:362] Metadata directory not provided
02:11:12.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.282886 24638 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:12.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.283285 24638 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:11:12.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.287608 24638 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/data/instance:
02:11:12.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9955c4e1a57041dfb205b833a12cd7dd"
02:11:12.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:12 on dist-test-slave-rnrw"
02:11:12.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.288757 24638 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/wal/instance:
02:11:12.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9955c4e1a57041dfb205b833a12cd7dd"
02:11:12.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:12 on dist-test-slave-rnrw"
02:11:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.296304 24638 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.003s
02:11:12.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.301992 24656 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.303104 24638 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:11:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.303406 24638 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/wal
02:11:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9955c4e1a57041dfb205b833a12cd7dd"
02:11:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:12 on dist-test-slave-rnrw"
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.303732 24638 fs_report.cc:389] FS layout report
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/wal
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/wal
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/data/data
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.358516 24638 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.360118 24638 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.360531 24638 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:12.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.363520 24638 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:11:12.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.368238 24638 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:11:12.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.368484 24638 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:11:12.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.368834 24638 ts_tablet_manager.cc:616] Registered 0 tablets
02:11:12.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.369019 24638 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:11:12.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.509042 24638 rpc_server.cc:307] RPC server started. Bound to: 127.23.151.130:42993
02:11:12.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.509188 24777 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.151.130:42993 every 8 connection(s)
02:11:12.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.512209 24638 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/data/info.pb
02:11:12.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.513101 24158 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 24638
02:11:12.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.513479 24158 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-1/wal/instance
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.552027 24158 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/wal
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/data
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/logs
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/data/info.pb
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.131:0
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.151.131
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.131
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:11:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.569963 24779 heartbeater.cc:344] Connected to a master server at 127.23.151.188:46563
02:11:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.570506 24779 heartbeater.cc:461] Registering TS with master...
02:11:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.571437 24778 heartbeater.cc:344] Connected to a master server at 127.23.151.189:40229
02:11:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.571810 24778 heartbeater.cc:461] Registering TS with master...
02:11:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.571996 24779 heartbeater.cc:507] Master 127.23.151.188:46563 requested a full tablet report, sending...
02:11:12.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.572979 24778 heartbeater.cc:507] Master 127.23.151.189:40229 requested a full tablet report, sending...
02:11:12.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.573853 24780 heartbeater.cc:344] Connected to a master server at 127.23.151.190:42715
02:11:12.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.574188 24780 heartbeater.cc:461] Registering TS with master...
02:11:12.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.575100 24780 heartbeater.cc:507] Master 127.23.151.190:42715 requested a full tablet report, sending...
02:11:12.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.575269 24375 ts_manager.cc:194] Registered new tserver with Master: 9955c4e1a57041dfb205b833a12cd7dd (127.23.151.130:42993)
02:11:12.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.576021 24287 ts_manager.cc:194] Registered new tserver with Master: 9955c4e1a57041dfb205b833a12cd7dd (127.23.151.130:42993)
02:11:12.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.577013 24375 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.151.130:42607
02:11:12.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.578095 24207 ts_manager.cc:194] Registered new tserver with Master: 9955c4e1a57041dfb205b833a12cd7dd (127.23.151.130:42993)
02:11:12.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.898307 24788 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:12.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.898801 24788 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.899302 24788 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:12.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.933703 24788 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:12.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.934466 24788 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.151.131
02:11:12.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.971488 24788 tablet_server_runner.cc:78] Tablet server non-default flags:
02:11:12.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/logs
02:11:12.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.151.148:36665
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/data
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/wal
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.151.131:0
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/data/info.pb
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.151.131
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.151.190:42715,127.23.151.189:40229,127.23.151.188:46563
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.151.131
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.972987 24788 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:12.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:12.974725 24788 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:12.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.987315 24794 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:13.581213 24779 heartbeater.cc:499] Master 127.23.151.188:46563 was elected leader, sending a full tablet report...
02:11:14.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:12.988821 24795 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:14.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:14.221630 24798 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:14.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:14.224330 24797 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1231 milliseconds
02:11:14.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.224454 24788 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:14.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.225703 24788 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.227979 24788 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.229329 24788 hybrid_clock.cc:648] HybridClock initialized: now 1786327874229300 us; error 64 us; skew 500 ppm
02:11:14.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.230253 24788 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:14.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.237119 24788 webserver.cc:533] Webserver started at http://127.23.151.131:33043/ using document root <none> and password file <none>
02:11:14.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.238014 24788 fs_manager.cc:362] Metadata directory not provided
02:11:14.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.238219 24788 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:14.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.238643 24788 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:11:14.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.243243 24788 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/data/instance:
02:11:14.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "02d938e08b4a4f88aefb36dc039941c2"
02:11:14.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:14 on dist-test-slave-rnrw"
02:11:14.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.244350 24788 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/wal/instance:
02:11:14.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "02d938e08b4a4f88aefb36dc039941c2"
02:11:14.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:14 on dist-test-slave-rnrw"
02:11:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.252097 24788 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
02:11:14.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.258118 24806 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.259346 24788 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.259734 24788 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/wal
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "02d938e08b4a4f88aefb36dc039941c2"
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:14 on dist-test-slave-rnrw"
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.260099 24788 fs_report.cc:389] FS layout report
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/wal
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/wal
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/data/data
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.318933 24788 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:14.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.320727 24788 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:14.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.321161 24788 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:14.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.324239 24788 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:11:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.328974 24788 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:11:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.329181 24788 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:11:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.329384 24788 ts_tablet_manager.cc:616] Registered 0 tablets
02:11:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.329523 24788 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:11:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.473084 24788 rpc_server.cc:307] RPC server started. Bound to: 127.23.151.131:46067
02:11:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.473218 24925 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.151.131:46067 every 8 connection(s)
02:11:14.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.475683 24788 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/data/info.pb
02:11:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.476763 24158 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 24788
02:11:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.477226 24158 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster1683560326961702745/ts-2/wal/instance
02:11:14.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.538777 24928 heartbeater.cc:344] Connected to a master server at 127.23.151.190:42715
02:11:14.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.539321 24928 heartbeater.cc:461] Registering TS with master...
02:11:14.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.541018 24928 heartbeater.cc:507] Master 127.23.151.190:42715 requested a full tablet report, sending...
02:11:14.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.541520 24926 heartbeater.cc:344] Connected to a master server at 127.23.151.189:40229
02:11:14.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.541806 24926 heartbeater.cc:461] Registering TS with master...
02:11:14.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.542680 24926 heartbeater.cc:507] Master 127.23.151.189:40229 requested a full tablet report, sending...
02:11:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.544756 24207 ts_manager.cc:194] Registered new tserver with Master: 02d938e08b4a4f88aefb36dc039941c2 (127.23.151.131:46067)
02:11:14.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.545409 24287 ts_manager.cc:194] Registered new tserver with Master: 02d938e08b4a4f88aefb36dc039941c2 (127.23.151.131:46067)
02:11:14.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.551232 24927 heartbeater.cc:344] Connected to a master server at 127.23.151.188:46563
02:11:14.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.551640 24927 heartbeater.cc:461] Registering TS with master...
02:11:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.552410 24927 heartbeater.cc:507] Master 127.23.151.188:46563 requested a full tablet report, sending...
02:11:14.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.554483 24375 ts_manager.cc:194] Registered new tserver with Master: 02d938e08b4a4f88aefb36dc039941c2 (127.23.151.131:46067)
02:11:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.555897 24375 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.151.131:41343
02:11:14.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.566949 24158 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
02:11:14.578 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:11:14.578 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:11:14.579 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.151.190"
port: 42715
}
bound_http_address {
host: "127.23.151.190"
port: 44561
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.151.189"
port: 40229
}
bound_http_address {
host: "127.23.151.189"
port: 40947
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.151.188"
port: 46563
}
bound_http_address {
host: "127.23.151.188"
port: 36565
}
}
}
02:11:14.580 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:11:14.581 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.151.129"
port: 32873
}
bound_http_address {
host: "127.23.151.129"
port: 33603
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.151.130"
port: 42993
}
bound_http_address {
host: "127.23.151.130"
port: 38553
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.151.131"
port: 46067
}
bound_http_address {
host: "127.23.151.131"
port: 33043
}
}
}
02:11:14.581 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:11:14.582 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster13677384733979241008/krb5kdc/krb5cc
02:11:14.582 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:11:14.583 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task5Jg9dI/test-tmp
02:11:14.584 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:11:14.584 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task5Jg9dI/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:11:14.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:11:14.963 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:14.963 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:11:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.966477 24938 mini_oidc.cc:150] Starting JWKS server
02:11:14.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.969560 24938 webserver.cc:533] Webserver started at http://127.0.0.1:36317/ using document root <none> and password file <none>
02:11:14.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.969956 24938 mini_oidc.cc:186] Starting OIDC Discovery server
02:11:14.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:14.971544 24938 webserver.cc:533] Webserver started at http://127.0.0.1:35305/ using document root <none> and password file <none>
02:11:15.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:11:15.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/principal' for realm 'KRBTEST.COM',
02:11:15.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:11:15.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:15 dist-test-slave-rnrw krb5kdc[24949](info): setting up network...
02:11:15.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:11:15.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:15 dist-test-slave-rnrw krb5kdc[24949](info): set up 2 sockets
02:11:15.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:15 dist-test-slave-rnrw krb5kdc[24949](info): commencing operation
02:11:15.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:11:15.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:15.558977 24927 heartbeater.cc:499] Master 127.23.151.188:46563 was elected leader, sending a full tablet report...
02:11:17.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.135581 24938 mini_kdc.cc:121] Time spent starting KDC: real 2.126s user 0.005s sys 0.016s
02:11:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:11:17.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:11:17.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:11:17.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:11:17.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:11:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:11:17.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:11:17.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:11:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:11:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/test-user.keytab.
02:11:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/test-user.keytab.
02:11:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:17 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327877, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:11:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T02:11:17Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:11:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T02:11:17Z Disabled control of system clock
02:11:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.90.190@KRBTEST.COM; defaulting to no policy
02:11:17.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:17.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.90.190@KRBTEST.COM" created.
02:11:17.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:17.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.90.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:17.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.90.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:17.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.24.90.190@KRBTEST.COM; defaulting to no policy
02:11:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.24.90.190@KRBTEST.COM" created.
02:11:17.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:17.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.90.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:17.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.90.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:17.385368 24938 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/logs
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data/info.pb
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.190:41189
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.190
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.190
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:17.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.718063 24968 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.718734 24968 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:17.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.719413 24968 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.755272 24968 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.755578 24968 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:11:17.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.756067 24968 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:17.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.756446 24968 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:17.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.756793 24968 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:11:17.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.757092 24968 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.757332 24968 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36317/jwks.json
02:11:17.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:17.798332 24968 master_runner.cc:387] Master server non-default flags:
02:11:17.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/logs
02:11:17.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:17.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:17.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:17.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.190
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.190:41189
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data/info.pb
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.190
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:17.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:17.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:17.800061 24968 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:17.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:17.801307 24968 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:17.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.811765 24974 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.814702 24975 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:17.817775 24977 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.042244 24976 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:11:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.042339 24968 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.046449 24968 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:19.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.049537 24968 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:19.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.050968 24968 hybrid_clock.cc:648] HybridClock initialized: now 1786327879050900 us; error 77 us; skew 500 ppm
02:11:19.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.051759 24968 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:19.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:19 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327879, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:19.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.057715 24968 init.cc:377] Logged in from keytab as kudu/127.24.90.190@KRBTEST.COM (short username kudu)
02:11:19.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.063589 24968 webserver.cc:533] Webserver started at http://127.24.90.190:45099/ using document root <none> and password file <none>
02:11:19.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.064476 24968 fs_manager.cc:362] Metadata directory not provided
02:11:19.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.064837 24968 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:19.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.065518 24968 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:11:19.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.070405 24968 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data/instance:
02:11:19.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5905cf16052445f5bfb3e4bb179c27a0"
02:11:19.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:19 on dist-test-slave-rnrw"
02:11:19.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.071619 24968 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal/instance:
02:11:19.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5905cf16052445f5bfb3e4bb179c27a0"
02:11:19.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:19 on dist-test-slave-rnrw"
02:11:19.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.079361 24968 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
02:11:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.086287 24986 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.087549 24968 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.087852 24968 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5905cf16052445f5bfb3e4bb179c27a0"
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:19 on dist-test-slave-rnrw"
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.088181 24968 fs_report.cc:389] FS layout report
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data/data
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:19.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:19.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:19.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.154846 24968 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:19.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.160269 24968 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:19.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.160831 24968 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:19.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.236377 24968 rpc_server.cc:307] RPC server started. Bound to: 127.24.90.190:41189
02:11:19.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.236593 25046 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.90.190:41189 every 8 connection(s)
02:11:19.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.239154 24968 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data/info.pb
02:11:19.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.245380 25048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:11:19.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.249464 24938 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 24968
02:11:19.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.250051 24938 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal/instance
02:11:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.261500 25048 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } has no permanent_uuid. Determining permanent_uuid...
02:11:19.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.90.189@KRBTEST.COM; defaulting to no policy
02:11:19.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:19.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.90.189@KRBTEST.COM" created.
02:11:19.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:19 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327879, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.190@KRBTEST.COM for kudu/127.24.90.190@KRBTEST.COM
02:11:19.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:19.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.90.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:19.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.90.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:19.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.303359 25048 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } has no permanent_uuid. Determining permanent_uuid...
02:11:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.306105 24989 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.310688 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.189:38975: Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:19.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.24.90.189@KRBTEST.COM; defaulting to no policy
02:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.24.90.189@KRBTEST.COM" created.
02:11:19.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:19.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.90.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:19.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.90.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.340845 24938 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/logs
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data/info.pb
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.189:38975
02:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.189
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.189
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.360188 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } attempt: 1
02:11:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.365118 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.189:38975: Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:19.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.433491 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } attempt: 2
02:11:19.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.437813 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.189:38975: Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:19.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.529201 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } attempt: 3
02:11:19.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.533497 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.189:38975: Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.662571 25057 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.663123 25057 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.663686 25057 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.676926 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } attempt: 4
02:11:19.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.681468 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.189:38975: Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.707278 25057 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.707628 25057 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:11:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.708079 25057 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.708401 25057 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.708752 25057 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:11:19.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.709084 25057 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:19.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.709321 25057 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36317/jwks.json
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.759480 25057 master_runner.cc:387] Master server non-default flags:
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/logs
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.189
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.189:38975
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data/info.pb
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.189
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:19.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:19.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:19.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:19.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.761220 25057 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:19.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.762280 25057 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:19.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.772831 25066 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:19.980942 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } attempt: 5
02:11:19.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.989305 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.189:38975: Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:20.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:20.536973 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } attempt: 6
02:11:20.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:20.548269 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.189:38975: Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:21.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:19.774511 25067 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.103865 25069 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:21.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.106673 25068 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1329 milliseconds
02:11:21.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.106787 25057 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:21.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.108417 25057 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.115479 25057 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.117095 25057 hybrid_clock.cc:648] HybridClock initialized: now 1786327881117044 us; error 76 us; skew 500 ppm
02:11:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.117897 25057 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:21 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327881, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:21.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.124078 25057 init.cc:377] Logged in from keytab as kudu/127.24.90.189@KRBTEST.COM (short username kudu)
02:11:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.129491 25057 webserver.cc:533] Webserver started at http://127.24.90.189:36439/ using document root <none> and password file <none>
02:11:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.130434 25057 fs_manager.cc:362] Metadata directory not provided
02:11:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.130628 25057 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.131018 25057 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:11:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.135403 25057 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data/instance:
02:11:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31e4dcc903a24dd184c86b29371de4f2"
02:11:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:21 on dist-test-slave-rnrw"
02:11:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.136473 25057 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal/instance:
02:11:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31e4dcc903a24dd184c86b29371de4f2"
02:11:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:21 on dist-test-slave-rnrw"
02:11:21.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.144074 25057 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
02:11:21.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.150260 25082 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.151954 25057 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.005s
02:11:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.152458 25057 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal
02:11:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31e4dcc903a24dd184c86b29371de4f2"
02:11:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:21 on dist-test-slave-rnrw"
02:11:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.153012 25057 fs_report.cc:389] FS layout report
02:11:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal
02:11:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal
02:11:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data/data
02:11:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.205505 25057 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:21.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.210956 25057 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:21.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.211395 25057 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.287425 25057 rpc_server.cc:307] RPC server started. Bound to: 127.24.90.189:38975
02:11:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.287569 25135 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.90.189:38975 every 8 connection(s)
02:11:21.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.290287 25057 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data/info.pb
02:11:21.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.296859 25137 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:11:21.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.300565 24938 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 25057
02:11:21.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.301079 24938 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal/instance
02:11:21.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.312369 25137 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } has no permanent_uuid. Determining permanent_uuid...
02:11:21.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.90.188@KRBTEST.COM; defaulting to no policy
02:11:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.90.188@KRBTEST.COM" created.
02:11:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:21 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327881, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.189@KRBTEST.COM for kudu/127.24.90.190@KRBTEST.COM
02:11:21.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.350122 25137 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } has no permanent_uuid. Determining permanent_uuid...
02:11:21.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:21.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.90.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:21.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.90.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:21.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:21 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327881, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.189@KRBTEST.COM for kudu/127.24.90.189@KRBTEST.COM
02:11:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.24.90.188@KRBTEST.COM; defaulting to no policy
02:11:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.24.90.188@KRBTEST.COM" created.
02:11:21.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.378273 25137 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } has no permanent_uuid. Determining permanent_uuid...
02:11:21.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.380919 25084 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.386021 25137 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:21.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:21.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.90.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:21.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.90.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab.
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.400905 24938 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/logs
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data/info.pb
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.188:46723
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.188
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.188
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.435524 25137 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 1
02:11:21.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.439908 25137 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:21.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.508317 25137 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 2
02:11:21.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.513522 25137 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.604948 25137 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 3
02:11:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.608886 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } attempt: 7
02:11:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.609689 25137 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:21.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:21 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327879, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.190@KRBTEST.COM for kudu/127.24.90.189@KRBTEST.COM
02:11:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.633641 25048 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } has no permanent_uuid. Determining permanent_uuid...
02:11:21.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.639748 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:21.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.698189 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 1
02:11:21.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.703819 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:21.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.749169 25147 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:21.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.749712 25147 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:21.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.750320 25147 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:21.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.753132 25137 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 4
02:11:21.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.758020 25137 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:21.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.785238 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 2
02:11:21.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.785815 25147 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:21.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.786099 25147 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:11:21.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.786501 25147 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:21.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.786818 25147 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:21.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.787155 25147 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:11:21.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.787521 25147 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:21.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.787794 25147 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36317/jwks.json
02:11:21.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.789896 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.826298 25147 master_runner.cc:387] Master server non-default flags:
02:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/logs
02:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data
02:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.188
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.188:46723
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data/info.pb
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.188
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:21.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.827953 25147 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:21.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.829052 25147 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.839725 25157 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:21.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:21.875228 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 3
02:11:21.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.883534 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:22.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:22.024096 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 4
02:11:22.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:22.031939 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:22.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:22.057446 25137 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 5
02:11:22.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:22.065457 25137 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:22.315503 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 5
02:11:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:22.323407 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:22.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:22.612983 25137 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 6
02:11:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:22.657980 25137 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:22.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:22.875936 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 6
02:11:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:22.913914 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:23.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.718588 25137 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 7
02:11:23.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:23.783803 25137 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:23.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:21.841547 25158 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:23.245985 25156 debug-util.cc:398] Leaking SignalData structure 0x7b0800028cc0 after lost signal to thread 25147
02:11:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:23.793803 25156 kernel_stack_watchdog.cc:198] Thread 25147 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
02:11:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:11:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:11:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:11:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:11:23.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:23.808223 25159 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1951 milliseconds
02:11:23.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:23.809561 25147 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.969s user 0.717s sys 1.218s
02:11:23.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:23.809958 25147 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.970s user 0.717s sys 1.218s
02:11:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:23.812320 25160 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.812409 25147 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.814137 25147 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:23.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.818317 25147 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.819782 25147 hybrid_clock.cc:648] HybridClock initialized: now 1786327883819686 us; error 73 us; skew 500 ppm
02:11:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.821218 25147 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:23.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:23 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327883, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.828977 25147 init.cc:377] Logged in from keytab as kudu/127.24.90.188@KRBTEST.COM (short username kudu)
02:11:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.833587 25147 webserver.cc:533] Webserver started at http://127.24.90.188:34421/ using document root <none> and password file <none>
02:11:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.834590 25147 fs_manager.cc:362] Metadata directory not provided
02:11:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.834839 25147 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.835354 25147 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:11:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.839730 25147 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data/instance:
02:11:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a677c8f5e1574778a7c8fa65d4a4eefd"
02:11:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:23 on dist-test-slave-rnrw"
02:11:23.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.841044 25147 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal/instance:
02:11:23.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a677c8f5e1574778a7c8fa65d4a4eefd"
02:11:23.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:23 on dist-test-slave-rnrw"
02:11:23.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.848810 25147 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
02:11:23.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.855105 25177 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.856762 25147 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
02:11:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.857200 25147 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal
02:11:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a677c8f5e1574778a7c8fa65d4a4eefd"
02:11:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:23 on dist-test-slave-rnrw"
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.857591 25147 fs_report.cc:389] FS layout report
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data/data
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:23.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.917593 25147 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:23.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.924021 25147 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:23.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.924619 25147 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:23.947460 25048 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } attempt: 7
02:11:23.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:23.954394 25048 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.24.90.188:46723: Network error: Client connection negotiation failed: client connection to 127.24.90.188:46723: connect: Connection refused (error 111)
02:11:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.012030 25147 rpc_server.cc:307] RPC server started. Bound to: 127.24.90.188:46723
02:11:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.012130 25234 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.90.188:46723 every 8 connection(s)
02:11:24.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.015278 25147 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data/info.pb
02:11:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.020539 24938 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 25147
02:11:24.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.021081 24938 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal/instance
02:11:24.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.022339 25235 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:11:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.035061 25235 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } has no permanent_uuid. Determining permanent_uuid...
02:11:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:24 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327883, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.188@KRBTEST.COM for kudu/127.24.90.190@KRBTEST.COM
02:11:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:24 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327877, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.90.188@KRBTEST.COM
02:11:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.071051 25235 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } has no permanent_uuid. Determining permanent_uuid...
02:11:24.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:24 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327883, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.188@KRBTEST.COM for kudu/127.24.90.189@KRBTEST.COM
02:11:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:24 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327877, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.90.189@KRBTEST.COM
02:11:24.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.098023 25235 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } has no permanent_uuid. Determining permanent_uuid...
02:11:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:24 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327883, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.188@KRBTEST.COM for kudu/127.24.90.188@KRBTEST.COM
02:11:24.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:24 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327877, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.90.190@KRBTEST.COM
02:11:24.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.128104 25235 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: Bootstrap starting.
02:11:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.133499 24938 external_mini_cluster.cc:955] 0 TS(s) registered with all masters
02:11:24.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.134656 25235 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: Neither blocks nor log segments found. Creating new log.
02:11:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.136483 25235 log.cc:826] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: Log is configured to *not* fsync() on all Append() calls
02:11:24.141 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:24.141 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:11:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.141831 25235 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: No bootstrap required, opened a new log
02:11:24.142 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:11:24.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:11:24.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.90.190"
port: 41189
}
bound_http_address {
host: "127.24.90.190"
port: 45099
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.90.189"
port: 38975
}
bound_http_address {
host: "127.24.90.189"
port: 36439
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.90.188"
port: 46723
}
bound_http_address {
host: "127.24.90.188"
port: 34421
}
}
}
02:11:24.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:11:24.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:24.144 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:11:24.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:11:24.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.160867 25235 raft_consensus.cc:359] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:24.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.161767 25235 raft_consensus.cc:385] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:11:24.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.162081 25235 raft_consensus.cc:740] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a677c8f5e1574778a7c8fa65d4a4eefd, State: Initialized, Role: FOLLOWER
02:11:24.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.162976 25235 consensus_queue.cc:260] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.166227 25247 sys_catalog.cc:455] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } } }
02:11:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.167089 25247 sys_catalog.cc:458] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [sys.catalog]: This master's current role is: FOLLOWER
02:11:24.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.168053 25235 sys_catalog.cc:565] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [sys.catalog]: configured and running, proceeding with master startup.
02:11:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:24 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327884, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:11:24.178 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:24.179 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5cc
02:11:24.181 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:11:24.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:24.181557 25184 tablet.cc:2406] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:11:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:24.197625 25259 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:24.197943 25259 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:24.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.193732 25147 master_runner.cc:187] Error getting master registration for 127.24.90.190:41189: OK, instance_id { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" instance_seqno: 1786327879215384 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:11:24.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.225522 25147 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:11:24.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.340773 25247 raft_consensus.cc:493] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:24.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.341394 25247 raft_consensus.cc:515] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:24.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:24.344753 25247 leader_election.cc:290] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189), 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975)
02:11:25.181 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.90.188:46723
02:11:25.182 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:11:25.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:25.182862 24938 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu with pid 25147
02:11:25.223 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:25.223 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.90.190:41189
02:11:25.223 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:11:25.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:25.224051 24938 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu with pid 24968
02:11:25.259 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:25.259 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.90.189:38975
02:11:25.259 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:11:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:25.259652 24938 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu with pid 25057
02:11:25.285 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:25.285 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.90.188:46723
02:11:25.285 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:25.287880 24938 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/logs
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data/info.pb
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.188:46723
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.188
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34421
02:11:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.188
02:11:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:25.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:25.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.617348 25266 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.617949 25266 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.618573 25266 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:25.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.654150 25266 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:25.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.654386 25266 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:11:25.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.654664 25266 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.654865 25266 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.655061 25266 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:11:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.655342 25266 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.655632 25266 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36317/jwks.json
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:25.697710 25266 master_runner.cc:387] Master server non-default flags:
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/logs
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.188
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.188:46723
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data/info.pb
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.188
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34421
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:25.699497 25266 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:25.700893 25266 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:25.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.714058 25273 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:27.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:25.715186 25274 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.095680 25276 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:27.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.099403 25275 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1379 milliseconds
02:11:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.100797 25266 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.386s user 0.488s sys 0.846s
02:11:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.101163 25266 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.387s user 0.488s sys 0.846s
02:11:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.101457 25266 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:27.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.103078 25266 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:27.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.106449 25266 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.107939 25266 hybrid_clock.cc:648] HybridClock initialized: now 1786327887107890 us; error 51 us; skew 500 ppm
02:11:27.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.109207 25266 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:27.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:27 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327887, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:27.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.118639 25266 init.cc:377] Logged in from keytab as kudu/127.24.90.188@KRBTEST.COM (short username kudu)
02:11:27.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.125415 25266 webserver.cc:533] Webserver started at http://127.24.90.188:34421/ using document root <none> and password file <none>
02:11:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.126722 25266 fs_manager.cc:362] Metadata directory not provided
02:11:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.127055 25266 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.139252 25266 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.004s sys 0.004s
02:11:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.145427 25284 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.146970 25266 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
02:11:27.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.147424 25266 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal
02:11:27.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a677c8f5e1574778a7c8fa65d4a4eefd"
02:11:27.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:23 on dist-test-slave-rnrw"
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.150453 25266 fs_report.cc:389] FS layout report
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/wal
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data/data
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.239616 25266 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:27.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.250759 25266 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:27.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.251194 25266 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:27.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.326777 25266 rpc_server.cc:307] RPC server started. Bound to: 127.24.90.188:46723
02:11:27.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.326817 25340 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.90.188:46723 every 8 connection(s)
02:11:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.330468 25266 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-2/data/info.pb
02:11:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.340973 24938 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 25266
02:11:27.341 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:27.341 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.90.190:41189
02:11:27.341 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:11:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.342422 25341 sys_catalog.cc:263] Verifying existing consensus state
02:11:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.343528 24938 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal
02:11:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/logs
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data/info.pb
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.190:41189
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.190
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45099
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.190
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.351083 25341 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: Bootstrap starting.
02:11:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.387323 25341 log.cc:826] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: Log is configured to *not* fsync() on all Append() calls
02:11:27.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.393361 25341 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:11:27.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.394426 25341 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: Bootstrap complete.
02:11:27.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.419857 25341 raft_consensus.cc:359] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:27.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.422271 25341 raft_consensus.cc:385] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:11:27.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.422515 25341 raft_consensus.cc:740] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a677c8f5e1574778a7c8fa65d4a4eefd, State: Initialized, Role: FOLLOWER
02:11:27.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.423405 25341 consensus_queue.cc:260] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:27.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.426119 25346 sys_catalog.cc:455] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } } }
02:11:27.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.426973 25346 sys_catalog.cc:458] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [sys.catalog]: This master's current role is: FOLLOWER
02:11:27.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.427289 25341 sys_catalog.cc:565] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [sys.catalog]: configured and running, proceeding with master startup.
02:11:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.460740 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.461210 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.457157 25266 master_runner.cc:187] Error getting master registration for 127.24.90.190:41189: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111),
02:11:27.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.499770 25266 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:11:27.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.506553 25290 tablet.cc:2406] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:11:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.529096 25346 raft_consensus.cc:493] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:27.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.529760 25346 raft_consensus.cc:515] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.535645 25286 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111)
02:11:27.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.543715 25287 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975): Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.541383 25346 leader_election.cc:290] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189), 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975)
02:11:27.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.547024 25286 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189): Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111)
02:11:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.548511 25286 leader_election.cc:304] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a677c8f5e1574778a7c8fa65d4a4eefd; no voters: 31e4dcc903a24dd184c86b29371de4f2, 5905cf16052445f5bfb3e4bb179c27a0
02:11:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.549755 25346 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:11:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.722648 25343 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.723337 25343 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:27.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.724067 25343 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.774991 25343 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.775293 25343 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:11:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.775710 25343 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.776048 25343 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.776424 25343 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:11:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.776701 25343 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:27.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.776891 25343 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36317/jwks.json
02:11:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.818468 25343 master_runner.cc:387] Master server non-default flags:
02:11:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/logs
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.190
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.190:41189
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data/info.pb
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.190
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45099
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.819968 25343 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:27.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:27.821195 25343 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:27.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.833143 25370 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:28.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:28.467697 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:28.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:28.468303 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:29.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:29.478267 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:29.481487 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:29.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:29.234781 25369 debug-util.cc:398] Leaking SignalData structure 0x7b0800028cc0 after lost signal to thread 25343
02:11:29.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:29.582950 25343 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.748s user 0.520s sys 0.880s
02:11:29.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:29.583384 25343 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.748s user 0.520s sys 0.880s
02:11:29.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:27.834985 25371 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:29.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:29.585513 25373 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:29.590181 25372 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1750 milliseconds
02:11:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.590193 25343 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:29.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.591578 25343 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:29.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.594230 25343 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:29.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.595619 25343 hybrid_clock.cc:648] HybridClock initialized: now 1786327889595563 us; error 62 us; skew 500 ppm
02:11:29.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.596510 25343 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:29.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:29 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327889, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.603192 25343 init.cc:377] Logged in from keytab as kudu/127.24.90.190@KRBTEST.COM (short username kudu)
02:11:29.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.609123 25343 webserver.cc:533] Webserver started at http://127.24.90.190:45099/ using document root <none> and password file <none>
02:11:29.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.610410 25343 fs_manager.cc:362] Metadata directory not provided
02:11:29.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.610715 25343 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:29.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.621795 25343 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.002s sys 0.006s
02:11:29.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.627635 25381 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.629043 25343 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:11:29.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.629467 25343 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal
02:11:29.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5905cf16052445f5bfb3e4bb179c27a0"
02:11:29.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:19 on dist-test-slave-rnrw"
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.632304 25343 fs_report.cc:389] FS layout report
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data/data
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:29.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.710908 25343 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.716904 25343 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.717458 25343 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:29.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.767464 25400 raft_consensus.cc:493] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:29.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.769331 25400 raft_consensus.cc:515] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:29.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.775503 25400 leader_election.cc:290] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189), 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975)
02:11:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:29.787367 25287 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975): Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:29.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:29.790071 25286 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189): Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111)
02:11:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.790647 25286 leader_election.cc:304] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a677c8f5e1574778a7c8fa65d4a4eefd; no voters: 31e4dcc903a24dd184c86b29371de4f2, 5905cf16052445f5bfb3e4bb179c27a0
02:11:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.791946 25400 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:11:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.840417 25343 rpc_server.cc:307] RPC server started. Bound to: 127.24.90.190:41189
02:11:29.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.840497 25440 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.90.190:41189 every 8 connection(s)
02:11:29.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.843529 25343 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/data/info.pb
02:11:29.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.850621 24938 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 25343
02:11:29.851 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:29.851 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.90.189:38975
02:11:29.851 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:11:29.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.853304 24938 external_mini_cluster.cc:1372] Running /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:29.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu
02:11:29.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal
02:11:29.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data
02:11:29.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:29.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:29.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:29.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/logs
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data/info.pb
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.189:38975
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.189
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36439
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.189
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:29.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.857051 25441 sys_catalog.cc:263] Verifying existing consensus state
02:11:29.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260810 02:11:29.858023 25441 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:11:29.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.859191 25343 master.cc:562] Master@127.24.90.190:41189 shutting down...
02:11:29.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:29.970367 25343 master.cc:584] Master@127.24.90.190:41189 shutdown complete.
02:11:29.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:29.973942 25343 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
02:11:29.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:11:30.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.227279 25443 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:11:30.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.227932 25443 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:30.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.228636 25443 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:11:30.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.264351 25443 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:30.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.264663 25443 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:11:30.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.264950 25443 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:11:30.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.265156 25443 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:30.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.265354 25443 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:11:30.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.265554 25443 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:30.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.265703 25443 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36317/jwks.json
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:30.307763 25443 master_runner.cc:387] Master server non-default flags:
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/logs
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.90.148:36315
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.90.190:41189,127.24.90.189:38975,127.24.90.188:46723
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/kudu.keytab
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.90.189
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.90.189:38975
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36317/jwks.json
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data/info.pb
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.90.189
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36439
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 4bcd631c449d3c6649730ca6e07660c8168f0d44
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Aug 2026 01:45:51 UTC on bdcb31816ec0
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13267
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:30.309149 25443 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:30.310276 25443 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:11:30.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.322326 25450 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:30.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.487202 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:30.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.487892 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:31.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:31.494285 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:31.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:31.494837 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:31.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:31.864452 24321 debug-util.cc:398] Leaking SignalData structure 0x7b08000977e0 after lost signal to thread 24251
02:11:31.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:31.865386 24321 debug-util.cc:398] Leaking SignalData structure 0x7b08000aa280 after lost signal to thread 24324
02:11:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.084838 25458 raft_consensus.cc:493] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:32.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.087883 25458 raft_consensus.cc:515] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:32.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.095865 25458 leader_election.cc:290] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189), 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975)
02:11:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.116127 25286 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189): Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111)
02:11:32.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.134577 25287 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975): Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:32.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.135288 25287 leader_election.cc:304] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a677c8f5e1574778a7c8fa65d4a4eefd; no voters: 31e4dcc903a24dd184c86b29371de4f2, 5905cf16052445f5bfb3e4bb179c27a0
02:11:32.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.136798 25458 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:11:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:31.725478 25449 debug-util.cc:398] Leaking SignalData structure 0x7b0800028cc0 after lost signal to thread 25443
02:11:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.158089 25449 kernel_stack_watchdog.cc:198] Thread 25443 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
02:11:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:11:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:11:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:11:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:11:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:30.324960 25451 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.159801 25443 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.836s user 0.605s sys 1.165s
02:11:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.160384 25453 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:32.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.160807 25443 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.838s user 0.606s sys 1.165s
02:11:32.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.161158 25443 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:32.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.160277 25452 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1836 milliseconds
02:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.166131 25443 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:32.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.175442 25443 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.177023 25443 hybrid_clock.cc:648] HybridClock initialized: now 1786327892176935 us; error 91 us; skew 500 ppm
02:11:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.177975 25443 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:32.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:32 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327892, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.185117 25443 init.cc:377] Logged in from keytab as kudu/127.24.90.189@KRBTEST.COM (short username kudu)
02:11:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.190889 25443 webserver.cc:533] Webserver started at http://127.24.90.189:36439/ using document root <none> and password file <none>
02:11:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.191957 25443 fs_manager.cc:362] Metadata directory not provided
02:11:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.192240 25443 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:32.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.201776 25443 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.005s
02:11:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.207260 25469 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:32.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.208711 25443 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.209117 25443 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data,/tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31e4dcc903a24dd184c86b29371de4f2"
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-10 02:11:21 on dist-test-slave-rnrw"
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.211182 25443 fs_report.cc:389] FS layout report
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data/data
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:32.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:32.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:32.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.271257 25443 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.287388 25443 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.288043 25443 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.381757 25443 rpc_server.cc:307] RPC server started. Bound to: 127.24.90.189:38975
02:11:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.381910 25525 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.90.189:38975 every 8 connection(s)
02:11:32.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.384894 25443 server_base.cc:1193] Dumped server information to /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/data/info.pb
02:11:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.395567 24938 external_mini_cluster.cc:1434] Started /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu as pid 25443
02:11:32.396 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:32.397 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.90.188:46723(127.24.90.188:46723)] connecting to peer
02:11:32.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.397806 25526 sys_catalog.cc:263] Verifying existing consensus state
02:11:32.399 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260810 02:11:32.399076 25526 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:11:32.401 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:32.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.400938 25443 master.cc:562] Master@127.24.90.189:38975 shutting down...
02:11:32.404 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.90.188:46723(127.24.90.188:46723)] Successfully connected to peer
02:11:32.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.415 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.24.90.189:38975(127.24.90.189:38975)] Successfully connected to peer
02:11:32.420 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
02:11:32.421 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5cc
02:11:32.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:32 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786327884, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:32.431 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:11:32.431 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
02:11:32.432 [DEBUG - kudu-nio-2] (SecurityContext.java:222) Refreshing Kerberos credentials...
02:11:32.432 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.24.90.188
02:11:32.432 [DEBUG - kudu-nio-2] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/krb5kdc/krb5cc
02:11:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:32 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786327884, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:32.441 [DEBUG - kudu-nio-2] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:11:32.441 [INFO - kudu-nio-2] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
02:11:32.442 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.24.90.189
02:11:32.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:32 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786327884, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.90.188@KRBTEST.COM
02:11:32.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:32 dist-test-slave-rnrw krb5kdc[24949](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786327884, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.90.189@KRBTEST.COM
02:11:32.481 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x4c5972a9, L:/127.0.0.1:41892 - R:/127.24.90.189:38975] using SASL/GSSAPI
02:11:32.488 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x92a93190, L:/127.0.0.1:46830 - R:/127.24.90.188:46723] using SASL/GSSAPI
02:11:32.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.499722 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:32.502 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Service unavailable: service kudu.master.MasterService not registered on Master
02:11:32.503 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.24.90.189:38975(127.24.90.189:38975)] handling channelInactive
02:11:32.503 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state READY due to: connection closed
02:11:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.503304 25266 master_runner.cc:187] Error getting master registration for 127.24.90.190:41189: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111),
02:11:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.503816 25266 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:11:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.500075 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:32.509 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:32.518 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:32.519 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:32.521 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.521 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.521 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.522 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.522 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.523 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.525 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:32.537 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:32.542 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:32.543 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.543 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.544 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.545 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.545 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.545 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.545 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:32.558 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:32.562 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:32.563 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.564 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.564 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.563 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.566 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.566 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.566 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:32.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:32.570878 25443 master.cc:584] Master@127.24.90.189:38975 shutdown complete.
02:11:32.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.574714 25443 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
02:11:32.577 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:32.578 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:32.579 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.579 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.579 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.579 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:32.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5Jg9dI/test-tmp/mini-kudu-cluster18332223356364590328/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:11:32.597 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:32.598 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:32.599 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.599 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.599 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.600 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.600 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.601 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:32.618 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:32.619 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.619 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:32.619 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.619 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.620 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.620 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.620 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:32.718 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:32.719 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.720 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.720 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.721 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:32.722 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.722 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.722 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:32.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.749640 24773 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad620 after lost signal to thread 24640
02:11:32.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:32.750782 24773 debug-util.cc:398] Leaking SignalData structure 0x7b08000adbc0 after lost signal to thread 24777
02:11:32.978 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:32.979 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:32.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:32.981 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.982 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.982 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:32.984 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:33.158 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:33.158 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:33.159 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:33.159 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:33.159 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:33.164 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:33.164 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:33.164 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:33.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:33.509302 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:33.509724 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:34.057 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:34.058 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:34.059 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:34.059 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:34.060 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:34.060 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:34.060 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:34.061 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:34.061 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:34.515369 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:34.515774 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:34.918 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:34.918 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:34.919 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:34.919 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:34.919 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:34.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:34.920 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:34.920 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:34.921 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:35.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:35.105933 25542 raft_consensus.cc:493] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:35.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:35.106331 25542 raft_consensus.cc:515] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:35.107954 25542 leader_election.cc:290] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189), 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975)
02:11:35.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:35.110051 25286 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111) [suppressed 5 similar messages]
02:11:35.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:35.113051 25286 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189): Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111)
02:11:35.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:35.113531 25287 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975): Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:35.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:35.113883 25287 leader_election.cc:304] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a677c8f5e1574778a7c8fa65d4a4eefd; no voters: 31e4dcc903a24dd184c86b29371de4f2, 5905cf16052445f5bfb3e4bb179c27a0
02:11:35.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:35.114400 25542 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:11:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:35.518975 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:35.519306 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:36.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:36.523072 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:36.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:36.523466 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:37.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:37 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327897, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:37.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:37.123232 25280 init.cc:295] Successfully reacquired a new kerberos TGT
02:11:37.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:37.507669 25266 master_runner.cc:187] Error getting master registration for 127.24.90.190:41189: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111),
02:11:37.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:37.508193 25266 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:11:37.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:37.526571 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:37.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:37.526834 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:37.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:37.723270 25549 raft_consensus.cc:493] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:37.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:37.723758 25549 raft_consensus.cc:515] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:37.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:37.725507 25549 leader_election.cc:290] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189), 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975)
02:11:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:37.730371 25286 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189): Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111)
02:11:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:37.731043 25287 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975): Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:37.731374 25287 leader_election.cc:304] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a677c8f5e1574778a7c8fa65d4a4eefd; no voters: 31e4dcc903a24dd184c86b29371de4f2, 5905cf16052445f5bfb3e4bb179c27a0
02:11:37.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:37.732082 25549 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:11:37.837 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:37.838 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:37.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:37.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:37.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:37.839 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:37.839 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:37.839 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:37.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:38.530007 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:38.530313 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:39.533349 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:39.533692 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:39.769472 24240 debug-util.cc:398] Leaking SignalData structure 0x7b08000a8180 after lost signal to thread 24172
02:11:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:39.770170 24240 debug-util.cc:398] Leaking SignalData structure 0x7b080009b8a0 after lost signal to thread 24243
02:11:40.378 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:40.378 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:40.379 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:40.379 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:40.379 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:40.380 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:40.380 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:40.380 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:40.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:40.478 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:40.482 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:40.482 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:40.482 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:40.483 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:40.484 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:40.484 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:40.484 [INFO - pool-44-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:40.484 [WARN - pool-44-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:40.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:40.538312 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:40.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:40.538691 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:40.855706 25555 raft_consensus.cc:493] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:40.856168 25555 raft_consensus.cc:515] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:40.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:40.858197 25555 leader_election.cc:290] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189), 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975)
02:11:40.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:40.860721 25286 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111) [suppressed 3 similar messages]
02:11:40.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:40.864315 25286 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189): Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111)
02:11:40.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:40.864925 25287 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975): Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:40.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:40.865330 25287 leader_election.cc:304] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a677c8f5e1574778a7c8fa65d4a4eefd; no voters: 31e4dcc903a24dd184c86b29371de4f2, 5905cf16052445f5bfb3e4bb179c27a0
02:11:40.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:40.865989 25555 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:11:41.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:41.542069 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:41.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:41.542333 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:42.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:42.511193 25266 master_runner.cc:187] Error getting master registration for 127.24.90.190:41189: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111),
02:11:42.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:42.511544 25266 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:11:42.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:42.545302 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:42.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:42.545634 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:43.549185 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:43.549484 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:44.098 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:44.099 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:44.099 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:44.099 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:44.099 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:44.100 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:44.100 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:44.101 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:44.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:44.218 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:44.219 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:44.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:44.220 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:44.220 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:44.221 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:44.221 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:44.221 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:44.221 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:44.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:44.553063 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:44.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:44.553356 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:45.556416 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:45.556768 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:46.537 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:46.538 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:46.538 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:46.538 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:46.538 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:46.539 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:46.539 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:46.539 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:46.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:46.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:46.560465 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:46.561129 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:47.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:47 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327907, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:47.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:47.126001 25280 init.cc:295] Successfully reacquired a new kerberos TGT
02:11:47.137 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:47.138 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:47.139 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:47.139 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:47.140 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:47.141 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:47.141 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:47.141 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:47.142 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:47.515652 25266 master_runner.cc:187] Error getting master registration for 127.24.90.190:41189: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111),
02:11:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:47.516172 25266 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:11:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:47.564430 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:47.564782 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:48.567912 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:48.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:48.568176 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:48.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:48.873368 25567 raft_consensus.cc:493] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:48.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:48.873992 25567 raft_consensus.cc:515] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:48.877524 25567 leader_election.cc:290] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189), 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975)
02:11:48.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:48.880461 25286 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:11:48.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:48.886503 25286 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189): Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111)
02:11:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:48.887072 25287 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975): Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:48.887377 25287 leader_election.cc:304] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a677c8f5e1574778a7c8fa65d4a4eefd; no voters: 31e4dcc903a24dd184c86b29371de4f2, 5905cf16052445f5bfb3e4bb179c27a0
02:11:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:48.887974 25567 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:11:49.477 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:49.478 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:49.479 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:49.479 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:49.479 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:49.480 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:49.480 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:49.480 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:49.482 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:49.571620 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:49.571961 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:49.997 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:49.998 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:49.998 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:49.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:49.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:49.999 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:49.999 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:50.000 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:50.000 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:50.358 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:50.358 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:50.359 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:50.359 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:50.359 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:50.359 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:50.360 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:50.360 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:50.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:50.575217 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:50.575527 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:51.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:51.578718 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:51.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:51.579044 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:52.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:52.519347 25266 master_runner.cc:187] Error getting master registration for 127.24.90.190:41189: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111),
02:11:52.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:52.519711 25266 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:11:52.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:52.582310 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:52.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:52.582661 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:53.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:53.587069 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:53.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:53.587404 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:53.638 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:53.638 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:53.639 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:53.639 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:53.639 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:53.639 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:53.639 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:53.639 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:53.641 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:54.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:54.590469 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:54.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:54.590734 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:55.593737 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:55.594007 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:55.877 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:55.878 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:55.878 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:55.878 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:55.878 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:55.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:55.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:55.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:55.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:11:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:56.597409 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:56.597751 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 10 02:11:57 dist-test-slave-rnrw krb5kdc[24949](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786327917, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.90.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:57.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:57.129014 25280 init.cc:295] Successfully reacquired a new kerberos TGT
02:11:57.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:57.523343 25266 master_runner.cc:187] Error getting master registration for 127.24.90.190:41189: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111),
02:11:57.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:57.523757 25266 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:11:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:57.600462 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:57.600736 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:58.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:58.113935 25586 raft_consensus.cc:493] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:58.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:58.114550 25586 raft_consensus.cc:515] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5905cf16052445f5bfb3e4bb179c27a0" member_type: VOTER last_known_addr { host: "127.24.90.190" port: 41189 } } peers { permanent_uuid: "31e4dcc903a24dd184c86b29371de4f2" member_type: VOTER last_known_addr { host: "127.24.90.189" port: 38975 } } peers { permanent_uuid: "a677c8f5e1574778a7c8fa65d4a4eefd" member_type: VOTER last_known_addr { host: "127.24.90.188" port: 46723 } }
02:11:58.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:58.116421 25586 leader_election.cc:290] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189), 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975)
02:11:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:58.118711 25286 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:11:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:58.121670 25287 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 31e4dcc903a24dd184c86b29371de4f2 (127.24.90.189:38975): Network error: Client connection negotiation failed: client connection to 127.24.90.189:38975: connect: Connection refused (error 111)
02:11:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:58.122205 25286 leader_election.cc:336] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5905cf16052445f5bfb3e4bb179c27a0 (127.24.90.190:41189): Network error: Client connection negotiation failed: client connection to 127.24.90.190:41189: connect: Connection refused (error 111)
02:11:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:58.122576 25286 leader_election.cc:304] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a677c8f5e1574778a7c8fa65d4a4eefd; no voters: 31e4dcc903a24dd184c86b29371de4f2, 5905cf16052445f5bfb3e4bb179c27a0
02:11:58.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:11:58.123212 25586 raft_consensus.cc:2749] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:11:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:58.603220 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:58.603511 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:59.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:59.606195 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:11:59.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:11:59.606487 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:11:59.618 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:11:59.619 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:11:59.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:59.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:59.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:11:59.620 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:59.620 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:59.621 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:11:59.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:12:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:12:00.278010 24773 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba960 after lost signal to thread 24640
02:12:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:12:00.278860 24773 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba0c0 after lost signal to thread 24777
02:12:00.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:12:00.609220 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:12:00.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:12:00.609530 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:12:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:12:01.612177 25359 catalog_manager.cc:1594] T 00000000000000000000000000000000 P a677c8f5e1574778a7c8fa65d4a4eefd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:12:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260810 02:12:01.612498 25359 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:12:02.258 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.190:41189(127.24.90.190:41189)] connecting to peer
02:12:02.259 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.24.90.189:38975(127.24.90.189:38975)] connecting to peer
02:12:02.259 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:12:02.259 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.24.90.190:41189(127.24.90.190:41189)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:12:02.260 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.190:41189: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189
02:12:02.260 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:12:02.260 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.24.90.189:38975(127.24.90.189:38975)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:12:02.260 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.24.90.189:38975: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
02:12:02.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975; will retry
02:12:02.263 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29865), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [13ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [106ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [113ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [113ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [125ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [127ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [129ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [130ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [145ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [147ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [149ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [149ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [166ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [168ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [170ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [170ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [183ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [183ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [186ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [186ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [203ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [205ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [205ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [205ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [223ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [224ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [225ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [225ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [321ms] refreshing cache from master, [321ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [324ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [324ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [325ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [326ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [326ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [581ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [584ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [586ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [588ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [588ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [761ms] refreshing cache from master, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [762ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [762ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [763ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [768ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [771ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [771ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [1660ms] refreshing cache from master, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [1662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [1665ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [2521ms] refreshing cache from master, [2521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [2522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [2522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [2523ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [2524ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [2524ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [2525ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [5441ms] refreshing cache from master, [5441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [5441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [5442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [5443ms] too many traces: truncated at 100 traces, deferred=Deferred@405580273(state=PENDING, result=null, callback=(continuation of Deferred@1085352792 after retry RPC after error@1298162020) -> (continuation of Deferred@1774972475 after retry RPC after error@341160402) -> (continuation of Deferred@1568125931 after retry RPC after error@332563040) -> (continuation of Deferred@635309079 after retry RPC after error@482140767) -> (continuation of Deferred@2083840712 after retry RPC after error@2071113388) -> (continuation of Deferred@781826913 after retry RPC after error@1439439607) -> (continuation of Deferred@238391332 after retry RPC after error@1616660608) -> (continuation of Deferred@58565977 after retry RPC after error@911412323) -> (continuation of Deferred@348462653 after retry RPC after error@383736941) -> (continuation of Deferred@2028369026 after retry RPC after error@1459113961) -> (continuation of Deferred@1785763837 after retry RPC after error@102798058) -> (continuation of Deferred@1302967496 after retry RPC after error@1496984979) -> (continuation of Deferred@924550710 after retry RPC after error@900658685) -> (continuation of Deferred@891357300 after retry RPC after error@1031798262) -> (continuation of Deferred@1455355481 after retry RPC after error@352182382) -> (continuation of Deferred@690104365 after retry RPC after error@1810948274) -> (continuation of Deferred@1247661995 after retry RPC after error@827893790) -> (continuation of Deferred@159620440 after retry RPC after error@1696209322) -> (continuation of Deferred@1100915253 after retry RPC after error@905116224) -> (continuation of Deferred@546232571 after retry RPC after error@367809033) -> (continuation of Deferred@559404427 after retry RPC after error@55905051) -> (continuation of Deferred@1439318666 after retry RPC after error@583089696) -> (continuation of Deferred@1034247294 after retry RPC after error@369329410) -> (continuation of Deferred@291134298 after retry RPC after error@985749359) -> (continuation of Deferred@764455653 after retry RPC after error@657523136), errback=(continuation of Deferred@1085352792 after retry RPC after error@1298162020) -> (continuation of Deferred@1774972475 after retry RPC after error@341160402) -> (continuation of Deferred@1568125931 after retry RPC after error@332563040) -> (continuation of Deferred@635309079 after retry RPC after error@482140767) -> (continuation of Deferred@2083840712 after retry RPC after error@2071113388) -> (continuation of Deferred@781826913 after retry RPC after error@1439439607) -> (continuation of Deferred@238391332 after retry RPC after error@1616660608) -> (continuation of Deferred@58565977 after retry RPC after error@911412323) -> (continuation of Deferred@348462653 after retry RPC after error@383736941) -> (continuation of Deferred@2028369026 after retry RPC after error@1459113961) -> (continuation of Deferred@1785763837 after retry RPC after error@102798058) -> (continuation of Deferred@1302967496 after retry RPC after error@1496984979) -> (continuation of Deferred@924550710 after retry RPC after error@900658685) -> (continuation of Deferred@891357300 after retry RPC after error@1031798262) -> (continuation of Deferred@1455355481 after retry RPC after error@352182382) -> (continuation of Deferred@690104365 after retry RPC after error@1810948274) -> (continuation of Deferred@1247661995 after retry RPC after error@827893790) -> (continuation of Deferred@159620440 after retry RPC after error@1696209322) -> (continuation of Deferred@1100915253 after retry RPC after error@905116224) -> (continuation of Deferred@546232571 after retry RPC after error@367809033) -> (continuation of Deferred@559404427 after retry RPC after error@55905051) -> (continuation of Deferred@1439318666 after retry RPC after error@583089696) -> (continuation of Deferred@1034247294 after retry RPC after error@369329410) -> (continuation of Deferred@291134298 after retry RPC after error@985749359) -> (continuation of Deferred@764455653 after retry RPC after error@657523136)))
02:12:02.264 [DEBUG - main] (Connection.java:575) [peer master-127.24.90.189:38975(127.24.90.189:38975)] disconnecting while in state TERMINATED
02:12:02.265 [DEBUG - main] (Connection.java:575) [peer master-127.24.90.188:46723(127.24.90.188:46723)] disconnecting while in state READY
02:12:02.265 [DEBUG - main] (Connection.java:575) [peer master-127.24.90.190:41189(127.24.90.190:41189)] disconnecting while in state TERMINATED
02:12:02.266 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:12:02.267 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.90.188:46723(127.24.90.188:46723)] handling channelInactive
02:12:02.267 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.90.188:46723(127.24.90.188:46723)] cleaning up while in state READY due to: connection closed
02:12:02.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:12:02.268771 24938 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu with pid 25266
02:12:03.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T02:12:03Z chronyd exiting
02:12:04.523 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:12:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:12:04.526094 24158 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu with pid 24484
02:12:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:12:04.559769 24158 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu with pid 24638
02:12:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:12:04.596809 24158 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu with pid 24788
02:12:04.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:12:04.629951 24158 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu with pid 24171
02:12:04.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:12:04.669947 24158 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu with pid 24246
02:12:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260810 02:12:04.708886 24158 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task5Jg9dI/build/tsan/bin/kudu with pid 24327
02:12:05.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-10T02:12:05Z chronyd exiting
02:12:06.916 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29865), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [13ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [106ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [113ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [113ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [125ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [127ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [129ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [130ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [145ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [147ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [149ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [149ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [166ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [168ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [170ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [170ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [183ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [183ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [186ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [186ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [203ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [205ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [205ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [205ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [223ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [224ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [225ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [225ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [321ms] refreshing cache from master, [321ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [324ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [324ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [325ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [326ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [326ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [581ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [584ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [586ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [588ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [588ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [761ms] refreshing cache from master, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [762ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [762ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [763ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [768ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [771ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [771ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [1660ms] refreshing cache from master, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [1662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [1665ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [2521ms] refreshing cache from master, [2521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [2522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [2522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [2523ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [2524ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [2524ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [2525ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [5441ms] refreshing cache from master, [5441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [5441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [5442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [5443ms] too many traces: truncated at 100 traces, deferred=Deferred@405580273(state=PENDING, result=null, callback=(continuation of Deferred@1085352792 after retry RPC after error@1298162020) -> (continuation of Deferred@1774972475 after retry RPC after error@341160402) -> (continuation of Deferred@1568125931 after retry RPC after error@332563040) -> (continuation of Deferred@635309079 after retry RPC after error@482140767) -> (continuation of Deferred@2083840712 after retry RPC after error@2071113388) -> (continuation of Deferred@781826913 after retry RPC after error@1439439607) -> (continuation of Deferred@238391332 after retry RPC after error@1616660608) -> (continuation of Deferred@58565977 after retry RPC after error@911412323) -> (continuation of Deferred@348462653 after retry RPC after error@383736941) -> (continuation of Deferred@2028369026 after retry RPC after error@1459113961) -> (continuation of Deferred@1785763837 after retry RPC after error@102798058) -> (continuation of Deferred@1302967496 after retry RPC after error@1496984979) -> (continuation of Deferred@924550710 after retry RPC after error@900658685) -> (continuation of Deferred@891357300 after retry RPC after error@1031798262) -> (continuation of Deferred@1455355481 after retry RPC after error@352182382) -> (continuation of Deferred@690104365 after retry RPC after error@1810948274) -> (continuation of Deferred@1247661995 after retry RPC after error@827893790) -> (continuation of Deferred@159620440 after retry RPC after error@1696209322) -> (continuation of Deferred@1100915253 after retry RPC after error@905116224) -> (continuation of Deferred@546232571 after retry RPC after error@367809033) -> (continuation of Deferred@559404427 after retry RPC after error@55905051) -> (continuation of Deferred@1439318666 after retry RPC after error@583089696) -> (continuation of Deferred@1034247294 after retry RPC after error@369329410) -> (continuation of Deferred@291134298 after retry RPC after error@985749359) -> (continuation of Deferred@764455653 after retry RPC after error@657523136), errback=(continuation of Deferred@1085352792 after retry RPC after error@1298162020) -> (continuation of Deferred@1774972475 after retry RPC after error@341160402) -> (continuation of Deferred@1568125931 after retry RPC after error@332563040) -> (continuation of Deferred@635309079 after retry RPC after error@482140767) -> (continuation of Deferred@2083840712 after retry RPC after error@2071113388) -> (continuation of Deferred@781826913 after retry RPC after error@1439439607) -> (continuation of Deferred@238391332 after retry RPC after error@1616660608) -> (continuation of Deferred@58565977 after retry RPC after error@911412323) -> (continuation of Deferred@348462653 after retry RPC after error@383736941) -> (continuation of Deferred@2028369026 after retry RPC after error@1459113961) -> (continuation of Deferred@1785763837 after retry RPC after error@102798058) -> (continuation of Deferred@1302967496 after retry RPC after error@1496984979) -> (continuation of Deferred@924550710 after retry RPC after error@900658685) -> (continuation of Deferred@891357300 after retry RPC after error@1031798262) -> (continuation of Deferred@1455355481 after retry RPC after error@352182382) -> (continuation of Deferred@690104365 after retry RPC after error@1810948274) -> (continuation of Deferred@1247661995 after retry RPC after error@827893790) -> (continuation of Deferred@159620440 after retry RPC after error@1696209322) -> (continuation of Deferred@1100915253 after retry RPC after error@905116224) -> (continuation of Deferred@546232571 after retry RPC after error@367809033) -> (continuation of Deferred@559404427 after retry RPC after error@55905051) -> (continuation of Deferred@1439318666 after retry RPC after error@583089696) -> (continuation of Deferred@1034247294 after retry RPC after error@369329410) -> (continuation of Deferred@291134298 after retry RPC after error@985749359) -> (continuation of Deferred@764455653 after retry RPC after error@657523136)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:348)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.90.189:38975
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29865), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [13ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [106ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [113ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [113ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [125ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [127ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [129ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [130ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [145ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [147ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [149ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [149ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [166ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [168ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [170ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [170ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [183ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [183ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [186ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [186ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [203ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [205ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [205ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [205ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [223ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [224ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [225ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [225ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [321ms] refreshing cache from master, [321ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [324ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [324ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [325ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [326ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [326ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [581ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [584ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [586ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [588ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [588ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [761ms] refreshing cache from master, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [762ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [762ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [763ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [768ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [771ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [771ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [1660ms] refreshing cache from master, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [1662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [1665ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [2521ms] refreshing cache from master, [2521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [2522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [2522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [2523ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.190:41189: Network error: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189, [2524ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.189:38975: Network error: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [2524ms] Sub RPC ConnectToMaster: received response from server master-127.24.90.188:46723: OK, [2525ms] delaying RPC due to: Service unavailable: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975, [5441ms] refreshing cache from master, [5441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.188:46723, [5441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.190:41189, [5442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.90.189:38975, [5443ms] too many traces: truncated at 100 traces, deferred=Deferred@405580273(state=PENDING, result=null, callback=(continuation of Deferred@1085352792 after retry RPC after error@1298162020) -> (continuation of Deferred@1774972475 after retry RPC after error@341160402) -> (continuation of Deferred@1568125931 after retry RPC after error@332563040) -> (continuation of Deferred@635309079 after retry RPC after error@482140767) -> (continuation of Deferred@2083840712 after retry RPC after error@2071113388) -> (continuation of Deferred@781826913 after retry RPC after error@1439439607) -> (continuation of Deferred@238391332 after retry RPC after error@1616660608) -> (continuation of Deferred@58565977 after retry RPC after error@911412323) -> (continuation of Deferred@348462653 after retry RPC after error@383736941) -> (continuation of Deferred@2028369026 after retry RPC after error@1459113961) -> (continuation of Deferred@1785763837 after retry RPC after error@102798058) -> (continuation of Deferred@1302967496 after retry RPC after error@1496984979) -> (continuation of Deferred@924550710 after retry RPC after error@900658685) -> (continuation of Deferred@891357300 after retry RPC after error@1031798262) -> (continuation of Deferred@1455355481 after retry RPC after error@352182382) -> (continuation of Deferred@690104365 after retry RPC after error@1810948274) -> (continuation of Deferred@1247661995 after retry RPC after error@827893790) -> (continuation of Deferred@159620440 after retry RPC after error@1696209322) -> (continuation of Deferred@1100915253 after retry RPC after error@905116224) -> (continuation of Deferred@546232571 after retry RPC after error@367809033) -> (continuation of Deferred@559404427 after retry RPC after error@55905051) -> (continuation of Deferred@1439318666 after retry RPC after error@583089696) -> (continuation of Deferred@1034247294 after retry RPC after error@369329410) -> (continuation of Deferred@291134298 after retry RPC after error@985749359) -> (continuation of Deferred@764455653 after retry RPC after error@657523136), errback=(continuation of Deferred@1085352792 after retry RPC after error@1298162020) -> (continuation of Deferred@1774972475 after retry RPC after error@341160402) -> (continuation of Deferred@1568125931 after retry RPC after error@332563040) -> (continuation of Deferred@635309079 after retry RPC after error@482140767) -> (continuation of Deferred@2083840712 after retry RPC after error@2071113388) -> (continuation of Deferred@781826913 after retry RPC after error@1439439607) -> (continuation of Deferred@238391332 after retry RPC after error@1616660608) -> (continuation of Deferred@58565977 after retry RPC after error@911412323) -> (continuation of Deferred@348462653 after retry RPC after error@383736941) -> (continuation of Deferred@2028369026 after retry RPC after error@1459113961) -> (continuation of Deferred@1785763837 after retry RPC after error@102798058) -> (continuation of Deferred@1302967496 after retry RPC after error@1496984979) -> (continuation of Deferred@924550710 after retry RPC after error@900658685) -> (continuation of Deferred@891357300 after retry RPC after error@1031798262) -> (continuation of Deferred@1455355481 after retry RPC after error@352182382) -> (continuation of Deferred@690104365 after retry RPC after error@1810948274) -> (continuation of Deferred@1247661995 after retry RPC after error@827893790) -> (continuation of Deferred@159620440 after retry RPC after error@1696209322) -> (continuation of Deferred@1100915253 after retry RPC after error@905116224) -> (continuation of Deferred@546232571 after retry RPC after error@367809033) -> (continuation of Deferred@559404427 after retry RPC after error@55905051) -> (continuation of Deferred@1439318666 after retry RPC after error@583089696) -> (continuation of Deferred@1034247294 after retry RPC after error@369329410) -> (continuation of Deferred@291134298 after retry RPC after error@985749359) -> (continuation of Deferred@764455653 after retry RPC after error@657523136)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:348)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.90.188:46723,127.24.90.190:41189,127.24.90.189:38975) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.190:41189(127.24.90.190:41189): Connection refused: /127.24.90.190:41189,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.90.189:38975(127.24.90.189:38975): Connection refused: /127.24.90.189:38975
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.90.189:38975
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)