19:52:39.192 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
19:52:39.205 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskkws3i4/test-tmp
19:52:39.207 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:52:39.211 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskkws3i4/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:52:39.447 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
19:52:39.593 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:52:39.594 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:52:39.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:39.597846 8763 mini_oidc.cc:152] Starting JWKS server
19:52:39.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:39.601722 8763 webserver.cc:460] Webserver started at http://127.0.0.1:44487/ using document root <none> and password file <none>
19:52:39.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:39.602258 8763 mini_oidc.cc:189] Starting OIDC Discovery server
19:52:39.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:39.604229 8763 webserver.cc:460] Webserver started at http://127.0.0.1:43155/ using document root <none> and password file <none>
19:52:39.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T19:52:39Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:52:39.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T19:52:39Z Disabled control of system clock
19:52:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:39.653666 8763 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/wal
19:52:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/data
19:52:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:52:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:52:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:52:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/logs
19:52:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/data/info.pb
19:52:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:52:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:52:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:52:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:52:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:52:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:52:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.254:35265
19:52:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.254
19:52:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44487/jwks.json with env {}
19:52:39.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:39.951311 8775 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:52:39.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:39.951925 8775 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:52:39.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:39.952561 8775 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:52:39.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:39.983410 8775 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:52:39.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:39.983724 8775 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:52:39.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:39.983942 8775 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:52:39.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:39.984151 8775 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:52:39.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:39.984431 8775 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:52:39.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:39.984658 8775 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44487/jwks.json
19:52:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:40.019694 8775 master_runner.cc:386] Master server non-default flags:
19:52:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:40.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:40.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/data
19:52:40.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/wal
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.254:35265
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44487/jwks.json
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/data/info.pb
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.254
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/logs
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:52:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:52:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:52:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:52:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:52:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:52:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:52:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:52:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:52:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:40.021139 8775 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:52:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:40.023352 8775 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:52:40.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:40.042362 8781 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:41.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:40.042392 8784 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:41.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:40.043485 8775 server_base.cc:1034] running on GCE node
19:52:41.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:40.042402 8782 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.224841 8775 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:52:41.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.228056 8775 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:52:41.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.229480 8775 hybrid_clock.cc:648] HybridClock initialized: now 1790538761229437 us; error 59 us; skew 500 ppm
19:52:41.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.230322 8775 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:52:41.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.243058 8775 webserver.cc:460] Webserver started at http://127.8.142.254:35309/ using document root <none> and password file <none>
19:52:41.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.244023 8775 fs_manager.cc:359] Metadata directory not provided
19:52:41.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.244210 8775 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:52:41.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.244580 8775 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:52:41.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.249375 8775 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/data/instance:
19:52:41.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d8c891aaf534cb0857a53092b83f716"
19:52:41.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:41 on dist-test-slave-zrlb"
19:52:41.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.250490 8775 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/wal/instance:
19:52:41.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d8c891aaf534cb0857a53092b83f716"
19:52:41.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:41 on dist-test-slave-zrlb"
19:52:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.257629 8775 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.004s
19:52:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.262954 8791 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:52:41.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.264178 8775 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:52:41.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.264446 8775 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/wal
19:52:41.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d8c891aaf534cb0857a53092b83f716"
19:52:41.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:41 on dist-test-slave-zrlb"
19:52:41.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.264724 8775 fs_report.cc:383] FS layout report
19:52:41.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:52:41.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/wal
19:52:41.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/wal
19:52:41.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/data/data
19:52:41.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:52:41.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:52:41.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:52:41.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:52:41.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:52:41.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:52:41.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:52:41.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:52:41.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:52:41.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:52:41.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:52:41.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.322032 8775 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:52:41.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.323484 8775 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:52:41.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.323972 8775 kserver.cc:163] Server-wide thread pool size limit: 3276
19:52:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.391750 8775 rpc_server.cc:307] RPC server started. Bound to: 127.8.142.254:35265
19:52:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.391992 8842 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.142.254:35265 every 8 connection(s)
19:52:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.394356 8775 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/data/info.pb
19:52:41.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.399659 8843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:52:41.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.404911 8763 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 8775
19:52:41.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.405467 8763 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-0/wal/instance
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.411412 8763 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/wal
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/data
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/logs
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/data/info.pb
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:52:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.253:42171
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.253
19:52:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:41.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:41.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44487/jwks.json with env {}
19:52:41.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.416409 8843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } has no permanent_uuid. Determining permanent_uuid...
19:52:41.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.447052 8843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } has no permanent_uuid. Determining permanent_uuid...
19:52:41.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.449299 8792 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.142.253:42171: connect: Connection refused (error 111)
19:52:41.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.453614 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.253:42171: Network error: Client connection negotiation failed: client connection to 127.8.142.253:42171: connect: Connection refused (error 111)
19:52:41.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.503211 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } attempt: 1
19:52:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.507298 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.253:42171: Network error: Client connection negotiation failed: client connection to 127.8.142.253:42171: connect: Connection refused (error 111)
19:52:41.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.575851 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } attempt: 2
19:52:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.580371 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.253:42171: Network error: Client connection negotiation failed: client connection to 127.8.142.253:42171: connect: Connection refused (error 111)
19:52:41.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.671769 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } attempt: 3
19:52:41.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.675287 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.253:42171: Network error: Client connection negotiation failed: client connection to 127.8.142.253:42171: connect: Connection refused (error 111)
19:52:41.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.727785 8845 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:52:41.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.728607 8845 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:52:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.729375 8845 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:52:41.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.761579 8845 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:52:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.761869 8845 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:52:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.762063 8845 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:52:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.762251 8845 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:52:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.762425 8845 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:52:41.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.762663 8845 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44487/jwks.json
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.797536 8845 master_runner.cc:386] Master server non-default flags:
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/data
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/wal
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.253:42171
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44487/jwks.json
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/data/info.pb
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.253
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:52:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:52:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:52:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/logs
19:52:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:52:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:52:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:52:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:52:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:52:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:52:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:52:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:52:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:52:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.798811 8845 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:52:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.800535 8845 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:52:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.817092 8855 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.818734 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } attempt: 4
19:52:41.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.822392 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.253:42171: Network error: Client connection negotiation failed: client connection to 127.8.142.253:42171: connect: Connection refused (error 111)
19:52:42.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:42.121757 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } attempt: 5
19:52:42.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:42.126236 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.253:42171: Network error: Client connection negotiation failed: client connection to 127.8.142.253:42171: connect: Connection refused (error 111)
19:52:42.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:42.673739 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } attempt: 6
19:52:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:42.682531 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.253:42171: Network error: Client connection negotiation failed: client connection to 127.8.142.253:42171: connect: Connection refused (error 111)
19:52:43.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.817094 8857 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:43.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:41.819118 8845 server_base.cc:1034] running on GCE node
19:52:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:41.817147 8854 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.024366 8845 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:52:43.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.027074 8845 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:52:43.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.028528 8845 hybrid_clock.cc:648] HybridClock initialized: now 1790538763028462 us; error 47 us; skew 500 ppm
19:52:43.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.029690 8845 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:52:43.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.035758 8845 webserver.cc:460] Webserver started at http://127.8.142.253:46285/ using document root <none> and password file <none>
19:52:43.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.036773 8845 fs_manager.cc:359] Metadata directory not provided
19:52:43.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.036954 8845 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:52:43.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.037410 8845 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:52:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.042208 8845 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/data/instance:
19:52:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e410c6c72b854495ac4436b1e8e18515"
19:52:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:43 on dist-test-slave-zrlb"
19:52:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.043314 8845 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/wal/instance:
19:52:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e410c6c72b854495ac4436b1e8e18515"
19:52:43.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:43 on dist-test-slave-zrlb"
19:52:43.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.050379 8845 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
19:52:43.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.055697 8866 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:52:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.056892 8845 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:52:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.057309 8845 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/wal
19:52:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e410c6c72b854495ac4436b1e8e18515"
19:52:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:43 on dist-test-slave-zrlb"
19:52:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.057787 8845 fs_report.cc:383] FS layout report
19:52:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:52:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/wal
19:52:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/wal
19:52:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/data/data
19:52:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:52:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:52:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:52:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:52:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:52:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:52:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:52:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:52:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:52:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:52:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:52:43.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.112483 8845 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:52:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.114235 8845 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:52:43.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.114789 8845 kserver.cc:163] Server-wide thread pool size limit: 3276
19:52:43.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.184741 8845 rpc_server.cc:307] RPC server started. Bound to: 127.8.142.253:42171
19:52:43.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.184836 8917 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.142.253:42171 every 8 connection(s)
19:52:43.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.187783 8845 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/data/info.pb
19:52:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.190541 8763 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 8845
19:52:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.191016 8763 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-1/wal/instance
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.193802 8763 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/wal
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/data
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/logs
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/data/info.pb
19:52:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.252:37465
19:52:43.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.252
19:52:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44487/jwks.json with env {}
19:52:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.194062 8918 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:52:43.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.207571 8918 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } has no permanent_uuid. Determining permanent_uuid...
19:52:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.234305 8918 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } has no permanent_uuid. Determining permanent_uuid...
19:52:43.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.251653 8918 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } has no permanent_uuid. Determining permanent_uuid...
19:52:43.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.253813 8868 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:43.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.257748 8918 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:43.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.307258 8918 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 1
19:52:43.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.312083 8918 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.380457 8918 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 2
19:52:43.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.384696 8918 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:43.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.476158 8918 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 3
19:52:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.480005 8918 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.509320 8920 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:52:43.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.509859 8920 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:52:43.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.510298 8920 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:52:43.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.540351 8920 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:52:43.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.540622 8920 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:52:43.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.540807 8920 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:52:43.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.540983 8920 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:52:43.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.541168 8920 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:52:43.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.541311 8920 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44487/jwks.json
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.575640 8920 master_runner.cc:386] Master server non-default flags:
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/data
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/wal
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.252:37465
19:52:43.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44487/jwks.json
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/data/info.pb
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.252
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/logs
19:52:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:52:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:52:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:52:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:52:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:52:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:52:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:52:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:52:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:52:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.576934 8920 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:52:43.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.578770 8920 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:52:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.598497 8931 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:43.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.623418 8918 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 4
19:52:43.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.628207 8918 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.742933 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } attempt: 7
19:52:43.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.763092 8843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } has no permanent_uuid. Determining permanent_uuid...
19:52:43.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.768980 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:43.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.827404 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 1
19:52:43.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.832101 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:43.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.913537 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 2
19:52:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.918423 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:43.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.927637 8918 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 5
19:52:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.932227 8918 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:44.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.003891 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 3
19:52:44.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:44.008565 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:44.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.149004 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 4
19:52:44.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:44.153826 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:44.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.437243 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 5
19:52:44.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:44.441491 8843 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.479578 8918 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 6
19:52:44.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:44.485977 8918 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.142.252:37465: Network error: Client connection negotiation failed: client connection to 127.8.142.252:37465: connect: Connection refused (error 111)
19:52:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.598518 8930 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:44.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:43.598526 8933 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:44.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:43.599900 8920 server_base.cc:1034] running on GCE node
19:52:44.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.814419 8920 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:52:44.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.816994 8920 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:52:44.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.818393 8920 hybrid_clock.cc:648] HybridClock initialized: now 1790538764818336 us; error 64 us; skew 500 ppm
19:52:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.819192 8920 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:52:44.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.826030 8920 webserver.cc:460] Webserver started at http://127.8.142.252:34211/ using document root <none> and password file <none>
19:52:44.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.827026 8920 fs_manager.cc:359] Metadata directory not provided
19:52:44.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.827266 8920 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:52:44.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.827890 8920 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:52:44.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.834923 8920 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/data/instance:
19:52:44.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "703101a4e91b48fd9b7a3fa6e0699d60"
19:52:44.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:44 on dist-test-slave-zrlb"
19:52:44.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.836432 8920 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/wal/instance:
19:52:44.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "703101a4e91b48fd9b7a3fa6e0699d60"
19:52:44.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:44 on dist-test-slave-zrlb"
19:52:44.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.844776 8920 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.003s sys 0.005s
19:52:44.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.851339 8944 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:52:44.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.852599 8920 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.852942 8920 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/wal
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "703101a4e91b48fd9b7a3fa6e0699d60"
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:44 on dist-test-slave-zrlb"
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.853245 8920 fs_report.cc:383] FS layout report
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/wal
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/wal
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/data/data
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:52:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:52:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:52:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:52:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:52:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:52:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:52:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:52:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:52:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:52:44.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.894197 8920 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:52:44.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.895814 8920 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:52:44.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.896386 8920 kserver.cc:163] Server-wide thread pool size limit: 3276
19:52:44.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.965137 8920 rpc_server.cc:307] RPC server started. Bound to: 127.8.142.252:37465
19:52:44.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.965237 8995 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.142.252:37465 every 8 connection(s)
19:52:44.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.968075 8920 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/data/info.pb
19:52:44.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.973354 8996 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:52:44.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.976883 8763 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 8920
19:52:44.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.977308 8763 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/master-2/wal/instance
19:52:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.989316 8996 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } has no permanent_uuid. Determining permanent_uuid...
19:52:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:44.993889 8843 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 6
19:52:45.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.031652 8996 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } has no permanent_uuid. Determining permanent_uuid...
19:52:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.040051 8843 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716: Bootstrap starting.
19:52:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.046720 8843 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716: Neither blocks nor log segments found. Creating new log.
19:52:45.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.048911 8843 log.cc:824] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716: Log is configured to *not* fsync() on all Append() calls
19:52:45.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.049386 8996 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } has no permanent_uuid. Determining permanent_uuid...
19:52:45.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.054845 8843 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716: No bootstrap required, opened a new log
19:52:45.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.070430 8996 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60: Bootstrap starting.
19:52:45.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.078122 8996 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60: Neither blocks nor log segments found. Creating new log.
19:52:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.080396 8996 log.cc:824] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60: Log is configured to *not* fsync() on all Append() calls
19:52:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.081439 8843 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } }
19:52:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.082346 8843 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:52:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.082639 8843 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0d8c891aaf534cb0857a53092b83f716, State: Initialized, Role: FOLLOWER
19:52:45.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.083567 8843 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } }
19:52:45.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.085527 8996 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60: No bootstrap required, opened a new log
19:52:45.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.087999 9008 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } } }
19:52:45.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.089221 9008 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [sys.catalog]: This master's current role is: FOLLOWER
19:52:45.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.089883 8843 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [sys.catalog]: configured and running, proceeding with master startup.
19:52:45.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.114464 8996 raft_consensus.cc:348] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } }
19:52:45.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.115345 8996 raft_consensus.cc:374] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:52:45.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.115628 8996 raft_consensus.cc:729] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 703101a4e91b48fd9b7a3fa6e0699d60, State: Initialized, Role: FOLLOWER
19:52:45.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.117041 8996 consensus_queue.cc:260] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } }
19:52:45.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.120626 9021 sys_catalog.cc:455] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } } }
19:52:45.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.121723 9021 sys_catalog.cc:458] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [sys.catalog]: This master's current role is: FOLLOWER
19:52:45.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.123046 8996 sys_catalog.cc:564] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [sys.catalog]: configured and running, proceeding with master startup.
19:52:45.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:45.135095 9019 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:52:45.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:45.135459 9019 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:52:45.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:45.151401 8950 tablet.cc:2369] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:52:45.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.115794 8775 master_runner.cc:186] Error getting master registration for 127.8.142.253:42171: OK, instance_id { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" instance_seqno: 1790538763165896 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:52:45.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.160006 8775 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:52:45.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:45.171654 9032 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:52:45.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:45.172415 9032 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:52:45.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.168354 8920 master_runner.cc:186] Error getting master registration for 127.8.142.253:42171: OK, instance_id { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" instance_seqno: 1790538763165896 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:52:45.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.220275 8920 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:52:45.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.275401 9008 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:52:45.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.276399 9008 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } }
19:52:45.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.280189 9008 leader_election.cc:288] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e410c6c72b854495ac4436b1e8e18515 (127.8.142.253:42171), 703101a4e91b48fd9b7a3fa6e0699d60 (127.8.142.252:37465)
19:52:45.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.281507 8971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0d8c891aaf534cb0857a53092b83f716" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" is_pre_election: true
19:52:45.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.282444 8971 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0d8c891aaf534cb0857a53092b83f716 in term 0.
19:52:45.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.285765 8793 leader_election.cc:302] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0d8c891aaf534cb0857a53092b83f716, 703101a4e91b48fd9b7a3fa6e0699d60; no voters:
19:52:45.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.287146 9008 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 0 FOLLOWER]: Leader pre-election won for term 1
19:52:45.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.288050 9008 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:52:45.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.289062 9008 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 0 FOLLOWER]: Advancing to term 1
19:52:45.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.296828 9008 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } }
19:52:45.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.300920 8971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0d8c891aaf534cb0857a53092b83f716" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "703101a4e91b48fd9b7a3fa6e0699d60"
19:52:45.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.301484 8971 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [term 0 FOLLOWER]: Advancing to term 1
19:52:45.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.309443 8971 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0d8c891aaf534cb0857a53092b83f716 in term 1.
19:52:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.316778 9008 leader_election.cc:288] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [CANDIDATE]: Term 1 election: Requested vote from peers e410c6c72b854495ac4436b1e8e18515 (127.8.142.253:42171), 703101a4e91b48fd9b7a3fa6e0699d60 (127.8.142.252:37465)
19:52:45.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.317030 8793 leader_election.cc:302] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0d8c891aaf534cb0857a53092b83f716, 703101a4e91b48fd9b7a3fa6e0699d60; no voters:
19:52:45.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.318578 9008 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 1 FOLLOWER]: Leader election won for term 1
19:52:45.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.324296 9008 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [term 1 LEADER]: Becoming Leader. State: Replica: 0d8c891aaf534cb0857a53092b83f716, State: Running, Role: LEADER
19:52:45.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.325871 9008 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } }
19:52:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:45.333036 8797 tablet.cc:2369] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:52:45.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.345291 9034 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0d8c891aaf534cb0857a53092b83f716. Latest consensus state: current_term: 1 leader_uuid: "0d8c891aaf534cb0857a53092b83f716" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } } }
19:52:45.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.347273 9034 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [sys.catalog]: This master's current role is: LEADER
19:52:45.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.351567 9036 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
19:52:45.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.358695 9036 catalog_manager.cc:1484] Initializing Kudu cluster ID...
19:52:45.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.386361 8971 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [term 1 FOLLOWER]: Refusing update from remote peer 0d8c891aaf534cb0857a53092b83f716: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:52:45.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.388484 9034 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [LEADER]: Connected to new peer: Peer: permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:52:45.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.443471 9021 sys_catalog.cc:455] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0d8c891aaf534cb0857a53092b83f716. Latest consensus state: current_term: 1 leader_uuid: "0d8c891aaf534cb0857a53092b83f716" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } } }
19:52:45.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.444316 9021 sys_catalog.cc:458] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [sys.catalog]: This master's current role is: FOLLOWER
19:52:45.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.453109 9037 mvcc.cc:205] Tried to move back new op lower bound from 7334046782997938176 to 7334046782784610304. Current Snapshot: MvccSnapshot[applied={T|T < 7334046782997938176}]
19:52:45.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.460683 9034 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0d8c891aaf534cb0857a53092b83f716" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } } }
19:52:45.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.461467 9034 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [sys.catalog]: This master's current role is: LEADER
19:52:45.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.463989 9036 catalog_manager.cc:1347] Generated new cluster ID: b117a3a37a6f4fb99069d03ba3b9c7e2
19:52:45.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.464272 9036 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
19:52:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.466023 9021 sys_catalog.cc:455] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0d8c891aaf534cb0857a53092b83f716" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } } }
19:52:45.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.466741 9021 sys_catalog.cc:458] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60 [sys.catalog]: This master's current role is: FOLLOWER
19:52:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.500437 9036 catalog_manager.cc:1370] Generated new certificate authority record
19:52:45.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.505208 9036 catalog_manager.cc:1504] Loading token signing keys...
19:52:45.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.531464 9036 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716: Generated new TSK 0
19:52:45.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.533473 9036 catalog_manager.cc:1514] Initializing in-progress tserver states...
19:52:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.546592 8918 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } attempt: 7
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.581496 8763 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/wal
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/data
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/logs
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/data/info.pb
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:52:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.193:0
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.142.193
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.193
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:45.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:52:45.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.597733 8918 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515: Bootstrap starting.
19:52:45.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.605507 8918 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515: Neither blocks nor log segments found. Creating new log.
19:52:45.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.608573 8918 log.cc:824] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515: Log is configured to *not* fsync() on all Append() calls
19:52:45.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.629501 8918 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515: No bootstrap required, opened a new log
19:52:45.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.665164 8918 raft_consensus.cc:348] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } }
19:52:45.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.666391 8918 raft_consensus.cc:374] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:52:45.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.666865 8918 raft_consensus.cc:729] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e410c6c72b854495ac4436b1e8e18515, State: Initialized, Role: FOLLOWER
19:52:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.668133 8918 consensus_queue.cc:260] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } }
19:52:45.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.692655 8918 sys_catalog.cc:564] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [sys.catalog]: configured and running, proceeding with master startup.
19:52:45.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.693274 8891 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [term 0 FOLLOWER]: Advancing to term 1
19:52:45.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.693773 8893 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0d8c891aaf534cb0857a53092b83f716" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e410c6c72b854495ac4436b1e8e18515" is_pre_election: true
19:52:45.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.694434 8892 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0d8c891aaf534cb0857a53092b83f716" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e410c6c72b854495ac4436b1e8e18515"
19:52:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.703365 8891 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [term 1 FOLLOWER]: Refusing update from remote peer 0d8c891aaf534cb0857a53092b83f716: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:52:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.704988 9034 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:52:45.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.707545 9046 raft_consensus.cc:482] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:52:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.689484 9045 sys_catalog.cc:455] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } } }
19:52:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.709156 9045 sys_catalog.cc:458] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [sys.catalog]: This master's current role is: FOLLOWER
19:52:45.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.708381 9046 raft_consensus.cc:504] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } }
19:52:45.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.722011 8818 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e410c6c72b854495ac4436b1e8e18515" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0d8c891aaf534cb0857a53092b83f716" is_pre_election: true
19:52:45.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.733175 8971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e410c6c72b854495ac4436b1e8e18515" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" is_pre_election: true
19:52:45.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.735139 8868 leader_election.cc:302] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e410c6c72b854495ac4436b1e8e18515; no voters: 0d8c891aaf534cb0857a53092b83f716, 703101a4e91b48fd9b7a3fa6e0699d60
19:52:45.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.745656 9046 leader_election.cc:288] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 0d8c891aaf534cb0857a53092b83f716 (127.8.142.254:35265), 703101a4e91b48fd9b7a3fa6e0699d60 (127.8.142.252:37465)
19:52:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.787976 9046 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
19:52:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.792176 9052 sys_catalog.cc:455] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0d8c891aaf534cb0857a53092b83f716" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } } }
19:52:45.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.794345 9052 sys_catalog.cc:458] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [sys.catalog]: This master's current role is: FOLLOWER
19:52:45.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.794404 9045 sys_catalog.cc:455] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0d8c891aaf534cb0857a53092b83f716. Latest consensus state: current_term: 1 leader_uuid: "0d8c891aaf534cb0857a53092b83f716" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } } }
19:52:45.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.796494 9045 sys_catalog.cc:458] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515 [sys.catalog]: This master's current role is: FOLLOWER
19:52:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:45.835027 9063 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
19:52:46.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:46.107313 9043 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:52:46.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:46.108029 9043 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:52:46.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:46.108807 9043 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:52:46.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:46.163880 9043 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:52:46.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:46.165251 9043 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.142.193
19:52:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.179003 9032 catalog_manager.cc:1259] Loaded cluster ID: b117a3a37a6f4fb99069d03ba3b9c7e2
19:52:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.179751 9032 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60: loading cluster ID for follower catalog manager: success
19:52:46.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.188316 9032 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60: acquiring CA information for follower catalog manager: success
19:52:46.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.194593 9032 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 703101a4e91b48fd9b7a3fa6e0699d60: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:52:46.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.236047 9043 tablet_server_runner.cc:78] Tablet server non-default flags:
19:52:46.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:46.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:46.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:46.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:46.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/data
19:52:46.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/wal
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.193:0
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/data/info.pb
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.193
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.142.193
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/logs
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:52:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:52:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:52:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:52:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:52:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:52:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:52:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:52:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.237560 9043 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:52:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.239565 9043 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:52:46.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:46.273622 9069 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:46.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.329888 9034 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0d8c891aaf534cb0857a53092b83f716" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0d8c891aaf534cb0857a53092b83f716" member_type: VOTER last_known_addr { host: "127.8.142.254" port: 35265 } } peers { permanent_uuid: "e410c6c72b854495ac4436b1e8e18515" member_type: VOTER last_known_addr { host: "127.8.142.253" port: 42171 } } peers { permanent_uuid: "703101a4e91b48fd9b7a3fa6e0699d60" member_type: VOTER last_known_addr { host: "127.8.142.252" port: 37465 } } }
19:52:46.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.330672 9034 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0d8c891aaf534cb0857a53092b83f716 [sys.catalog]: This master's current role is: LEADER
19:52:46.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.868410 9063 catalog_manager.cc:1259] Loaded cluster ID: b117a3a37a6f4fb99069d03ba3b9c7e2
19:52:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.869118 9063 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515: loading cluster ID for follower catalog manager: success
19:52:46.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.886883 9063 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515: acquiring CA information for follower catalog manager: success
19:52:46.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:46.893513 9063 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e410c6c72b854495ac4436b1e8e18515: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:52:47.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:47.666937 9068 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 9043
19:52:47.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:47.864588 9068 kernel_stack_watchdog.cc:198] Thread 9043 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 392ms:
19:52:47.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
19:52:47.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
19:52:47.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
19:52:47.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
19:52:47.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:47.868357 9072 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:47.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:47.876127 9043 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.603s user 0.487s sys 0.732s
19:52:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:47.877712 9043 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.605s user 0.488s sys 0.733s
19:52:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:46.274606 9070 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:47.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.881613 9043 server_base.cc:1034] running on GCE node
19:52:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.883195 9043 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:52:47.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.894757 9043 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:52:47.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.902712 9043 hybrid_clock.cc:648] HybridClock initialized: now 1790538767902621 us; error 90 us; skew 500 ppm
19:52:47.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.905000 9043 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:52:47.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.934255 9043 webserver.cc:460] Webserver started at http://127.8.142.193:43409/ using document root <none> and password file <none>
19:52:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.935894 9043 fs_manager.cc:359] Metadata directory not provided
19:52:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.936374 9043 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:52:47.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.937094 9043 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:52:47.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.944609 9043 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/data/instance:
19:52:47.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "176b061e7b73475080f9a1ee3be9d01c"
19:52:47.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:47 on dist-test-slave-zrlb"
19:52:47.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.946444 9043 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/wal/instance:
19:52:47.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "176b061e7b73475080f9a1ee3be9d01c"
19:52:47.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:47 on dist-test-slave-zrlb"
19:52:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.961882 9043 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.012s sys 0.000s
19:52:47.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.974946 9082 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:52:47.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.976579 9043 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.006s sys 0.004s
19:52:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.977169 9043 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/wal
19:52:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "176b061e7b73475080f9a1ee3be9d01c"
19:52:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:47 on dist-test-slave-zrlb"
19:52:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:47.977880 9043 fs_report.cc:383] FS layout report
19:52:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:52:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/wal
19:52:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/wal
19:52:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/data/data
19:52:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:52:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:52:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:52:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:52:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:52:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:52:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:52:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:52:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:52:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:52:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:52:48.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.055104 9043 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:52:48.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.057408 9043 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:52:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.058849 9043 kserver.cc:163] Server-wide thread pool size limit: 3276
19:52:48.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.071581 9043 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:52:48.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.088851 9043 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:52:48.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.089526 9043 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
19:52:48.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.090072 9043 ts_tablet_manager.cc:610] Registered 0 tablets
19:52:48.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.090420 9043 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:52:48.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.519949 9043 rpc_server.cc:307] RPC server started. Bound to: 127.8.142.193:33531
19:52:48.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.520033 9194 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.142.193:33531 every 8 connection(s)
19:52:48.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.523453 9043 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/data/info.pb
19:52:48.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.528650 8763 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 9043
19:52:48.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.529110 8763 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-0/wal/instance
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.543470 8763 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/wal
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/data
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/logs
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/data/info.pb
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:52:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.194:0
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.142.194
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.194
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:52:48.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.627662 9195 heartbeater.cc:347] Connected to a master server at 127.8.142.252:37465
19:52:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.628172 9195 heartbeater.cc:464] Registering TS with master...
19:52:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.629315 9195 heartbeater.cc:511] Master 127.8.142.252:37465 requested a full tablet report, sending...
19:52:48.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.631052 9196 heartbeater.cc:347] Connected to a master server at 127.8.142.253:42171
19:52:48.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.631402 9196 heartbeater.cc:464] Registering TS with master...
19:52:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.634395 8961 ts_manager.cc:194] Registered new tserver with Master: 176b061e7b73475080f9a1ee3be9d01c (127.8.142.193:33531)
19:52:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.635860 9196 heartbeater.cc:511] Master 127.8.142.253:42171 requested a full tablet report, sending...
19:52:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.638779 8883 ts_manager.cc:194] Registered new tserver with Master: 176b061e7b73475080f9a1ee3be9d01c (127.8.142.193:33531)
19:52:48.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.649119 9200 heartbeater.cc:347] Connected to a master server at 127.8.142.254:35265
19:52:48.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.649482 9200 heartbeater.cc:464] Registering TS with master...
19:52:48.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.650341 9200 heartbeater.cc:511] Master 127.8.142.254:35265 requested a full tablet report, sending...
19:52:48.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.652841 8808 ts_manager.cc:194] Registered new tserver with Master: 176b061e7b73475080f9a1ee3be9d01c (127.8.142.193:33531)
19:52:48.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.655632 8808 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.142.193:33227
19:52:48.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:48.924190 9199 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:52:48.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:48.924778 9199 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:52:48.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:48.925477 9199 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:52:48.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:48.957592 9199 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:52:48.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:48.958961 9199 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.142.194
19:52:48.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.995246 9199 tablet_server_runner.cc:78] Tablet server non-default flags:
19:52:48.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:48.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:48.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/data
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/wal
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.194:0
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/data/info.pb
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.194
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.142.194
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/logs
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:52:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:52:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:52:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:52:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:52:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:52:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:52:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:52:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:52:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.996845 9199 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:52:48.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:48.998935 9199 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:52:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:49.025589 9212 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:49.660527 9200 heartbeater.cc:503] Master 127.8.142.254:35265 was elected leader, sending a full tablet report...
19:52:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:49.025662 9214 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:49.026885 9199 server_base.cc:1034] running on GCE node
19:52:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:49.027668 9211 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.247903 9199 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:52:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.250458 9199 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:52:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.251900 9199 hybrid_clock.cc:648] HybridClock initialized: now 1790538770251847 us; error 66 us; skew 500 ppm
19:52:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.252748 9199 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:52:50.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.259138 9199 webserver.cc:460] Webserver started at http://127.8.142.194:35053/ using document root <none> and password file <none>
19:52:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.260140 9199 fs_manager.cc:359] Metadata directory not provided
19:52:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.260330 9199 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:52:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.260704 9199 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:52:50.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.265411 9199 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/data/instance:
19:52:50.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6f770512efd4ef38ed6483b6449971a"
19:52:50.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:50 on dist-test-slave-zrlb"
19:52:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.266422 9199 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/wal/instance:
19:52:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6f770512efd4ef38ed6483b6449971a"
19:52:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:50 on dist-test-slave-zrlb"
19:52:50.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.273563 9199 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
19:52:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.279407 9223 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:52:50.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.280656 9199 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:52:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.281059 9199 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/wal
19:52:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6f770512efd4ef38ed6483b6449971a"
19:52:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:50 on dist-test-slave-zrlb"
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.281461 9199 fs_report.cc:383] FS layout report
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/wal
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/wal
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/data/data
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:52:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:52:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.322314 9199 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:52:50.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.323967 9199 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:52:50.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.324507 9199 kserver.cc:163] Server-wide thread pool size limit: 3276
19:52:50.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.327158 9199 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:52:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.331476 9199 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:52:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.331712 9199 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:52:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.332099 9199 ts_tablet_manager.cc:610] Registered 0 tablets
19:52:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.332284 9199 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:52:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.481654 9199 rpc_server.cc:307] RPC server started. Bound to: 127.8.142.194:38961
19:52:50.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.481827 9335 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.142.194:38961 every 8 connection(s)
19:52:50.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.484371 9199 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/data/info.pb
19:52:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.494666 8763 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 9199
19:52:50.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.495493 8763 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-1/wal/instance
19:52:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.511135 8763 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/wal
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/data
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/logs
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/data/info.pb
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:52:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.195:0
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.142.195
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.195
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:52:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.532096 9337 heartbeater.cc:347] Connected to a master server at 127.8.142.253:42171
19:52:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.532588 9337 heartbeater.cc:464] Registering TS with master...
19:52:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.533797 9336 heartbeater.cc:347] Connected to a master server at 127.8.142.252:37465
19:52:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.533767 9337 heartbeater.cc:511] Master 127.8.142.253:42171 requested a full tablet report, sending...
19:52:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.534149 9336 heartbeater.cc:464] Registering TS with master...
19:52:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.534976 9336 heartbeater.cc:511] Master 127.8.142.252:37465 requested a full tablet report, sending...
19:52:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.536942 8883 ts_manager.cc:194] Registered new tserver with Master: d6f770512efd4ef38ed6483b6449971a (127.8.142.194:38961)
19:52:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.537608 8961 ts_manager.cc:194] Registered new tserver with Master: d6f770512efd4ef38ed6483b6449971a (127.8.142.194:38961)
19:52:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.544201 9338 heartbeater.cc:347] Connected to a master server at 127.8.142.254:35265
19:52:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.544646 9338 heartbeater.cc:464] Registering TS with master...
19:52:50.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.545671 9338 heartbeater.cc:511] Master 127.8.142.254:35265 requested a full tablet report, sending...
19:52:50.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.548023 8808 ts_manager.cc:194] Registered new tserver with Master: d6f770512efd4ef38ed6483b6449971a (127.8.142.194:38961)
19:52:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.549602 8808 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.142.194:42301
19:52:50.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:50.852445 9346 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:52:50.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:50.852911 9346 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:52:50.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:50.853344 9346 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:52:50.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:50.885414 9346 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:52:50.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:50.886231 9346 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.142.195
19:52:50.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.922513 9346 tablet_server_runner.cc:78] Tablet server non-default flags:
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.142.212:39463
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/data
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/wal
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.142.195:0
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/data/info.pb
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.142.195
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.142.254:35265,127.8.142.253:42171,127.8.142.252:37465
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.142.195
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/logs
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:52:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:52:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:52:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:52:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:52:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:52:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:52:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:52:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:52:50.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.924002 9346 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:52:50.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.926144 9346 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:52:50.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:50.946267 9352 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:51.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:51.554016 9338 heartbeater.cc:503] Master 127.8.142.254:35265 was elected leader, sending a full tablet report...
19:52:52.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:50.946267 9355 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:52.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:50.948725 9346 server_base.cc:1034] running on GCE node
19:52:52.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:50.946421 9353 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.154528 9346 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:52:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.156925 9346 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:52:52.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.158352 9346 hybrid_clock.cc:648] HybridClock initialized: now 1790538772158311 us; error 52 us; skew 500 ppm
19:52:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.159348 9346 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:52:52.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.166196 9346 webserver.cc:460] Webserver started at http://127.8.142.195:43157/ using document root <none> and password file <none>
19:52:52.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.167171 9346 fs_manager.cc:359] Metadata directory not provided
19:52:52.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.167410 9346 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:52:52.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.168031 9346 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:52:52.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.173059 9346 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/data/instance:
19:52:52.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c1a04ab7d2d343e0bb1c8036fe528b78"
19:52:52.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:52 on dist-test-slave-zrlb"
19:52:52.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.174376 9346 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/wal/instance:
19:52:52.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c1a04ab7d2d343e0bb1c8036fe528b78"
19:52:52.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:52 on dist-test-slave-zrlb"
19:52:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.182180 9346 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
19:52:52.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.188169 9363 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:52:52.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.189400 9346 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
19:52:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.189875 9346 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/wal
19:52:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c1a04ab7d2d343e0bb1c8036fe528b78"
19:52:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:52 on dist-test-slave-zrlb"
19:52:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.190263 9346 fs_report.cc:383] FS layout report
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/wal
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/wal
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/data/data
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:52:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:52:52.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.234118 9346 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:52:52.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.235915 9346 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:52:52.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.236411 9346 kserver.cc:163] Server-wide thread pool size limit: 3276
19:52:52.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.239148 9346 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:52:52.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.243206 9346 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:52:52.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.243426 9346 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:52:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.243758 9346 ts_tablet_manager.cc:610] Registered 0 tablets
19:52:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.244005 9346 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.001s
19:52:52.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.377405 9346 rpc_server.cc:307] RPC server started. Bound to: 127.8.142.195:39959
19:52:52.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.377521 9475 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.142.195:39959 every 8 connection(s)
19:52:52.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.380149 9346 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/data/info.pb
19:52:52.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.382413 8763 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 9346
19:52:52.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.382936 8763 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster3851707191757519444/ts-2/wal/instance
19:52:52.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.442773 9476 heartbeater.cc:347] Connected to a master server at 127.8.142.252:37465
19:52:52.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.443235 9476 heartbeater.cc:464] Registering TS with master...
19:52:52.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.444497 9476 heartbeater.cc:511] Master 127.8.142.252:37465 requested a full tablet report, sending...
19:52:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.447079 9477 heartbeater.cc:347] Connected to a master server at 127.8.142.253:42171
19:52:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.447402 9477 heartbeater.cc:464] Registering TS with master...
19:52:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.447736 8961 ts_manager.cc:194] Registered new tserver with Master: c1a04ab7d2d343e0bb1c8036fe528b78 (127.8.142.195:39959)
19:52:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.448232 9477 heartbeater.cc:511] Master 127.8.142.253:42171 requested a full tablet report, sending...
19:52:52.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.451061 8883 ts_manager.cc:194] Registered new tserver with Master: c1a04ab7d2d343e0bb1c8036fe528b78 (127.8.142.195:39959)
19:52:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.454214 9478 heartbeater.cc:347] Connected to a master server at 127.8.142.254:35265
19:52:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.454526 9478 heartbeater.cc:464] Registering TS with master...
19:52:52.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.455294 9478 heartbeater.cc:511] Master 127.8.142.254:35265 requested a full tablet report, sending...
19:52:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.457537 8807 ts_manager.cc:194] Registered new tserver with Master: c1a04ab7d2d343e0bb1c8036fe528b78 (127.8.142.195:39959)
19:52:52.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.459239 8807 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.142.195:57777
19:52:52.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:52.461932 8763 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
19:52:52.472 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:52:52.473 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
19:52:52.484 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.142.254"
port: 35265
}
bound_http_address {
host: "127.8.142.254"
port: 35309
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.142.253"
port: 42171
}
bound_http_address {
host: "127.8.142.253"
port: 46285
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.142.252"
port: 37465
}
bound_http_address {
host: "127.8.142.252"
port: 34211
}
}
}
19:52:52.490 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:52:52.493 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.142.193"
port: 33531
}
bound_http_address {
host: "127.8.142.193"
port: 43409
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.142.194"
port: 38961
}
bound_http_address {
host: "127.8.142.194"
port: 35053
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.142.195"
port: 39959
}
bound_http_address {
host: "127.8.142.195"
port: 43157
}
}
}
19:52:52.493 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
19:52:52.674 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
19:52:52.790 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskkws3i4/test-tmp
19:52:52.791 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:52:52.791 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskkws3i4/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:52:52.794 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
19:52:53.119 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:52:53.119 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:52:53.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:53.122677 9487 mini_oidc.cc:152] Starting JWKS server
19:52:53.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:53.126313 9487 webserver.cc:460] Webserver started at http://127.0.0.1:37261/ using document root <none> and password file <none>
19:52:53.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:53.126715 9487 mini_oidc.cc:189] Starting OIDC Discovery server
19:52:53.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:53.128464 9487 webserver.cc:460] Webserver started at http://127.0.0.1:44705/ using document root <none> and password file <none>
19:52:53.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
19:52:53.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/principal' for realm 'KRBTEST.COM',
19:52:53.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
19:52:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:52:53 dist-test-slave-zrlb krb5kdc[9499](info): setting up network...
19:52:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
19:52:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:52:53 dist-test-slave-zrlb krb5kdc[9499](info): set up 2 sockets
19:52:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:52:53 dist-test-slave-zrlb krb5kdc[9499](info): commencing operation
19:52:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
19:52:53.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:53.463308 9478 heartbeater.cc:503] Master 127.8.142.254:35265 was elected leader, sending a full tablet report...
19:52:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.249895 9487 mini_kdc.cc:121] Time spent starting KDC: real 1.097s user 0.001s sys 0.019s
19:52:54.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
19:52:54.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:52:54.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
19:52:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
19:52:54.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:52:54.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
19:52:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
19:52:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:52:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
19:52:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:52:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/test-user.keytab.
19:52:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/test-user.keytab.
19:52:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:52:54 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:52:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:52:54.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T19:52:54Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:52:54.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T19:52:54Z Disabled control of system clock
19:52:54.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.67.254@KRBTEST.COM; defaulting to no policy
19:52:54.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:52:54.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.67.254@KRBTEST.COM" created.
19:52:54.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:52:54.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.67.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.254.keytab.
19:52:54.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.67.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.254.keytab.
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:54.445273 9487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/logs
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data/info.pb
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:52:54.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.254:38049
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.254
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.254.keytab
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.254
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:54.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:54.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:52:54.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:54.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:52:54.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:52:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.776166 9513 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:52:54.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.776983 9513 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:52:54.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.777899 9513 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:52:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.812700 9513 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:52:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.812984 9513 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:52:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.813380 9513 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:52:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.813702 9513 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:52:54.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.814013 9513 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:52:54.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.814329 9513 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:52:54.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.814559 9513 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37261/jwks.json
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:54.850941 9513 master_runner.cc:386] Master server non-default flags:
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.254.keytab
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.254
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.254:38049
19:52:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data/info.pb
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.254
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/logs
19:52:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:52:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:52:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:52:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:52:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:52:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:52:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:52:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:52:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:52:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:54.852454 9513 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:52:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:54.853813 9513 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:52:54.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.872251 9520 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.873268 9519 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:54.873831 9513 server_base.cc:1034] running on GCE node
19:52:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:54.872328 9522 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.112840 9513 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:52:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.115368 9513 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:52:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.116729 9513 hybrid_clock.cc:648] HybridClock initialized: now 1790538776116677 us; error 70 us; skew 500 ppm
19:52:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.117522 9513 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:52:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:52:56 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538776, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:52:56.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.123522 9513 init.cc:377] Logged in from keytab as kudu/127.9.67.254@KRBTEST.COM (short username kudu)
19:52:56.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.128713 9513 webserver.cc:460] Webserver started at http://127.9.67.254:44817/ using document root <none> and password file <none>
19:52:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.129585 9513 fs_manager.cc:359] Metadata directory not provided
19:52:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.129799 9513 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:52:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.130249 9513 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:52:56.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.134892 9513 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data/instance:
19:52:56.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5721abf73154c0abb3b3d6fe7e6f21c"
19:52:56.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:56 on dist-test-slave-zrlb"
19:52:56.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.136135 9513 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal/instance:
19:52:56.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5721abf73154c0abb3b3d6fe7e6f21c"
19:52:56.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:56 on dist-test-slave-zrlb"
19:52:56.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.143414 9513 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
19:52:56.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.148963 9531 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:52:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.150089 9513 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
19:52:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.150403 9513 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal
19:52:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5721abf73154c0abb3b3d6fe7e6f21c"
19:52:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:56 on dist-test-slave-zrlb"
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.150712 9513 fs_report.cc:383] FS layout report
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data/data
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:52:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:52:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:52:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:52:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:52:56.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.183270 9513 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:52:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.189445 9513 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:52:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.189965 9513 kserver.cc:163] Server-wide thread pool size limit: 3276
19:52:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.262537 9513 rpc_server.cc:307] RPC server started. Bound to: 127.9.67.254:38049
19:52:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.262622 9582 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.67.254:38049 every 8 connection(s)
19:52:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.265218 9513 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data/info.pb
19:52:56.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.268570 9487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 9513
19:52:56.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.268950 9487 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal/instance
19:52:56.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.273351 9583 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:52:56.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.67.253@KRBTEST.COM; defaulting to no policy
19:52:56.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.285147 9583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } has no permanent_uuid. Determining permanent_uuid...
19:52:56.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:52:56.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.67.253@KRBTEST.COM" created.
19:52:56.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:52:56 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538776, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.254@KRBTEST.COM for kudu/127.9.67.254@KRBTEST.COM
19:52:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:52:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.67.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.253.keytab.
19:52:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.67.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.253.keytab.
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.315858 9487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/logs
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data/info.pb
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:52:56.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.253:34749
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.253
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.253.keytab
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.253
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:56.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:52:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:52:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:52:56.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.327481 9583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } has no permanent_uuid. Determining permanent_uuid...
19:52:56.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.330602 9532 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:52:56.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.335213 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.253:34749: Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:52:56.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.384807 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } attempt: 1
19:52:56.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.388758 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.253:34749: Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:52:56.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.457209 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } attempt: 2
19:52:56.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.461058 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.253:34749: Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:52:56.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.552484 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } attempt: 3
19:52:56.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.556382 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.253:34749: Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:52:56.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.643100 9589 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:52:56.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.643966 9589 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:52:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.645083 9589 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:52:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.692233 9589 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:52:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.692513 9589 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:52:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.692901 9589 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:52:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.693226 9589 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:52:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.693526 9589 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:52:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.693845 9589 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:52:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.694072 9589 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37261/jwks.json
19:52:56.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.699776 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } attempt: 4
19:52:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.703855 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.253:34749: Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.732697 9589 master_runner.cc:386] Master server non-default flags:
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.253.keytab
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.253
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.253:34749
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:52:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data/info.pb
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.253
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/logs
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:52:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:52:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.733778 9589 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:52:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.734793 9589 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:52:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.752485 9597 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:57.003332 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } attempt: 5
19:52:57.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:57.008400 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.253:34749: Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:52:57.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:57.555858 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } attempt: 6
19:52:57.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:57.562392 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.253:34749: Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:52:57.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:57.666172 8839 debug-util.cc:398] Leaking SignalData structure 0x7b08000a3c60 after lost signal to thread 8776
19:52:57.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:57.666935 8839 debug-util.cc:398] Leaking SignalData structure 0x7b08000b06c0 after lost signal to thread 8842
19:52:58.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.753530 9599 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:58.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:56.754313 9589 server_base.cc:1034] running on GCE node
19:52:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:56.752512 9596 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:58.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.015987 9589 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:52:58.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.019155 9589 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:52:58.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.020839 9589 hybrid_clock.cc:648] HybridClock initialized: now 1790538778020803 us; error 56 us; skew 500 ppm
19:52:58.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.021816 9589 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:52:58.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:52:58 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538778, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:52:58.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.029421 9589 init.cc:377] Logged in from keytab as kudu/127.9.67.253@KRBTEST.COM (short username kudu)
19:52:58.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.035679 9589 webserver.cc:460] Webserver started at http://127.9.67.253:44605/ using document root <none> and password file <none>
19:52:58.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.036972 9589 fs_manager.cc:359] Metadata directory not provided
19:52:58.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.037245 9589 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:52:58.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.037812 9589 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:52:58.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.045477 9589 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data/instance:
19:52:58.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cad9fd7310c74087b8b3fc78f439d3ec"
19:52:58.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:58 on dist-test-slave-zrlb"
19:52:58.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.046947 9589 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal/instance:
19:52:58.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cad9fd7310c74087b8b3fc78f439d3ec"
19:52:58.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:58 on dist-test-slave-zrlb"
19:52:58.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.057101 9589 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.011s sys 0.001s
19:52:58.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.064704 9609 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:52:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.065971 9589 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.000s sys 0.004s
19:52:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.066290 9589 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal
19:52:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cad9fd7310c74087b8b3fc78f439d3ec"
19:52:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:58 on dist-test-slave-zrlb"
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.066617 9589 fs_report.cc:383] FS layout report
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data/data
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:52:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:52:58.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.109122 9589 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:52:58.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.115100 9589 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:52:58.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.115605 9589 kserver.cc:163] Server-wide thread pool size limit: 3276
19:52:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.221001 9589 rpc_server.cc:307] RPC server started. Bound to: 127.9.67.253:34749
19:52:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.221120 9660 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.67.253:34749 every 8 connection(s)
19:52:58.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.224831 9589 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data/info.pb
19:52:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.226632 9487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 9589
19:52:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.227090 9487 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal/instance
19:52:58.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.240320 9661 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:52:58.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.67.252@KRBTEST.COM; defaulting to no policy
19:52:58.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:52:58.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.67.252@KRBTEST.COM" created.
19:52:58.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.258109 9661 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } has no permanent_uuid. Determining permanent_uuid...
19:52:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:52:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.67.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.252.keytab.
19:52:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.67.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.252.keytab.
19:52:58.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.293571 9487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/logs
19:52:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data/info.pb
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.252:37603
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.252
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.252.keytab
19:52:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.252
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:52:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:52:58.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:52:58 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538778, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.253@KRBTEST.COM for kudu/127.9.67.254@KRBTEST.COM
19:52:58.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.326187 9661 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } has no permanent_uuid. Determining permanent_uuid...
19:52:58.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:52:58 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538778, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.253@KRBTEST.COM for kudu/127.9.67.253@KRBTEST.COM
19:52:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.362639 9661 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } has no permanent_uuid. Determining permanent_uuid...
19:52:58.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.365178 9612 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:58.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.369431 9661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.419000 9661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 1
19:52:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.423424 9661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.491855 9661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 2
19:52:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.496641 9661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.588093 9661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 3
19:52:58.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.592974 9661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.622838 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } attempt: 7
19:52:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:52:58 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538776, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.254@KRBTEST.COM for kudu/127.9.67.253@KRBTEST.COM
19:52:58.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.651664 9667 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:52:58.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.652769 9667 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:52:58.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.653627 9667 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:52:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.656155 9583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } has no permanent_uuid. Determining permanent_uuid...
19:52:58.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.662277 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:58.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.693121 9667 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:52:58.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.693336 9667 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:52:58.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.693604 9667 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:52:58.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.693799 9667 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:52:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.693981 9667 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:52:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.694168 9667 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:52:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.694305 9667 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37261/jwks.json
19:52:58.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.720712 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 1
19:52:58.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.725127 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.735540 9667 master_runner.cc:386] Master server non-default flags:
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.252.keytab
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.252
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.252:37603
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data/info.pb
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.252
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/logs
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:52:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.736421 9661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 4
19:52:58.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.738095 9667 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:52:58.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.739367 9667 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:52:58.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.740743 9661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:58.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.770677 9677 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:52:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.806594 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 2
19:52:58.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.814523 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:58.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:58.900092 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 3
19:52:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.906939 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:59.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:59.040238 9661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 5
19:52:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:59.046001 9661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:59.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:59.047518 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 4
19:52:59.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:59.052805 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:59.336249 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 5
19:52:59.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:59.341619 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:59.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:59.593597 9661 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 6
19:52:59.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:59.600764 9661 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:52:59.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:52:59.894104 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } attempt: 6
19:52:59.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:59.900669 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.67.252:37603: Network error: Client connection negotiation failed: client connection to 127.9.67.252:37603: connect: Connection refused (error 111)
19:53:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.772722 9678 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:00.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:52:58.771324 9680 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:00.058981 9679 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:53:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.059096 9667 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:53:00.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.062999 9667 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:53:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.065567 9667 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:53:00.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.066929 9667 hybrid_clock.cc:648] HybridClock initialized: now 1790538780066897 us; error 62 us; skew 500 ppm
19:53:00.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.067730 9667 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:53:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:00 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538780, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:53:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.073822 9667 init.cc:377] Logged in from keytab as kudu/127.9.67.252@KRBTEST.COM (short username kudu)
19:53:00.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.078725 9667 webserver.cc:460] Webserver started at http://127.9.67.252:44921/ using document root <none> and password file <none>
19:53:00.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.079711 9667 fs_manager.cc:359] Metadata directory not provided
19:53:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.080004 9667 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:53:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.080521 9667 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:53:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.085172 9667 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data/instance:
19:53:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1351e3549344f58b4465974d7e78f6e"
19:53:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:53:00 on dist-test-slave-zrlb"
19:53:00.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.086321 9667 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal/instance:
19:53:00.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1351e3549344f58b4465974d7e78f6e"
19:53:00.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:53:00 on dist-test-slave-zrlb"
19:53:00.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.094043 9667 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
19:53:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.099913 9696 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:53:00.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.101069 9667 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
19:53:00.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.101480 9667 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal
19:53:00.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1351e3549344f58b4465974d7e78f6e"
19:53:00.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:53:00 on dist-test-slave-zrlb"
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.101891 9667 fs_report.cc:383] FS layout report
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data/data
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:53:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:53:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.160435 9667 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:53:00.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.166471 9667 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:53:00.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.166879 9667 kserver.cc:163] Server-wide thread pool size limit: 3276
19:53:00.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.246572 9667 rpc_server.cc:307] RPC server started. Bound to: 127.9.67.252:37603
19:53:00.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.246640 9747 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.67.252:37603 every 8 connection(s)
19:53:00.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.249183 9667 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data/info.pb
19:53:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.254257 9748 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:53:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.259588 9487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 9667
19:53:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.260134 9487 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal/instance
19:53:00.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.269665 9748 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } has no permanent_uuid. Determining permanent_uuid...
19:53:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:00 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538780, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.252@KRBTEST.COM for kudu/127.9.67.254@KRBTEST.COM
19:53:00.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:00 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.67.252@KRBTEST.COM
19:53:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.311205 9748 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } has no permanent_uuid. Determining permanent_uuid...
19:53:00.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:00 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538780, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.252@KRBTEST.COM for kudu/127.9.67.253@KRBTEST.COM
19:53:00.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:00 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.67.253@KRBTEST.COM
19:53:00.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.336956 9748 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } has no permanent_uuid. Determining permanent_uuid...
19:53:00.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:00 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538780, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.252@KRBTEST.COM for kudu/127.9.67.252@KRBTEST.COM
19:53:00.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:00 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.67.254@KRBTEST.COM
19:53:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.367363 9748 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: Bootstrap starting.
19:53:00.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.373337 9748 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: Neither blocks nor log segments found. Creating new log.
19:53:00.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.373703 9487 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
19:53:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.375908 9748 log.cc:824] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: Log is configured to *not* fsync() on all Append() calls
19:53:00.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.381652 9748 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: No bootstrap required, opened a new log
19:53:00.384 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:53:00.386 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
19:53:00.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
19:53:00.399 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
19:53:00.403 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.9.67.254"
port: 38049
}
bound_http_address {
host: "127.9.67.254"
port: 44817
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.9.67.253"
port: 34749
}
bound_http_address {
host: "127.9.67.253"
port: 44605
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.9.67.252"
port: 37603
}
bound_http_address {
host: "127.9.67.252"
port: 44921
}
}
}
19:53:00.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:53:00.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:53:00.404 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:53:00.406 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:53:00.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.414790 9748 raft_consensus.cc:348] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:00.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.415561 9748 raft_consensus.cc:374] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:53:00.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.415752 9748 raft_consensus.cc:729] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d1351e3549344f58b4465974d7e78f6e, State: Initialized, Role: FOLLOWER
19:53:00.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.416677 9748 consensus_queue.cc:260] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:00.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.419982 9758 sys_catalog.cc:455] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } } }
19:53:00.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.420861 9758 sys_catalog.cc:458] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [sys.catalog]: This master's current role is: FOLLOWER
19:53:00.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.421284 9758 raft_consensus.cc:482] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:53:00.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.421908 9748 sys_catalog.cc:564] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [sys.catalog]: configured and running, proceeding with master startup.
19:53:00.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:00.421981 9702 tablet.cc:2369] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:53:00.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.423105 9758 raft_consensus.cc:504] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.426385 9758 leader_election.cc:288] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049), cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749)
19:53:00.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:00 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538780, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:53:00.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:53:00.445 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:53:00.447 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5cc
19:53:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:00.454959 9771 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:00.455298 9771 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:00.456 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:53:00.457 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5cc
19:53:00.459 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:53:00.463 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
19:53:00.465 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:53:00.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:53:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.451546 9667 master_runner.cc:186] Error getting master registration for 127.9.67.254:38049: OK, instance_id { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" instance_seqno: 1790538776241525 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:53:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.515951 9667 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:53:00.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:00 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538780, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:53:00.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:53:00.542 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:53:00.542 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5cc
19:53:00.544 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:53:00.544 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.67.254:38049
19:53:00.546 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
19:53:00.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.547291 9487 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu with pid 9513
19:53:00.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:00.581873 9700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.9.67.254:38049 (error 108)
19:53:00.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:53:00.594 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.67.252:37603
19:53:00.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
19:53:00.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:00.592954 9700 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049): Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111)
19:53:00.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.594662 9487 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu with pid 9667
19:53:00.625 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:53:00.625 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.67.253:34749
19:53:00.625 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
19:53:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.625880 9487 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu with pid 9589
19:53:00.645 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:53:00.645 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.67.254:38049
19:53:00.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:00.648768 9487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/logs
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data/info.pb
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:53:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:53:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:53:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:53:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:53:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:53:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:53:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:53:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:53:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.254:38049
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.254
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44817
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.254.keytab
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.254
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:53:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:53:00.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:53:00.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:53:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:00.968030 9776 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:53:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:00.968775 9776 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:53:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:00.969535 9776 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:53:01.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:01.004848 9776 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:53:01.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:01.005141 9776 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:53:01.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:01.005545 9776 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:53:01.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:01.005883 9776 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:53:01.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:01.006194 9776 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:53:01.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:01.006520 9776 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:53:01.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:01.006760 9776 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37261/jwks.json
19:53:01.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:01.043391 9776 master_runner.cc:386] Master server non-default flags:
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.254.keytab
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.254
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.254:38049
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data/info.pb
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.254
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44817
19:53:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:53:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:53:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:53:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:53:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:53:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/logs
19:53:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:53:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:53:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:53:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:53:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:53:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:53:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:53:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:53:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:53:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:53:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:01.044977 9776 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:53:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:01.046407 9776 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:53:01.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:01.064651 9786 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:01.065193 9783 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:01.066699 9776 server_base.cc:1034] running on GCE node
19:53:02.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:01.064741 9784 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.308352 9776 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:53:02.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.311080 9776 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:53:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.312547 9776 hybrid_clock.cc:648] HybridClock initialized: now 1790538782312517 us; error 40 us; skew 500 ppm
19:53:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.313508 9776 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:53:02.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:02 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538782, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:53:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.319521 9776 init.cc:377] Logged in from keytab as kudu/127.9.67.254@KRBTEST.COM (short username kudu)
19:53:02.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.325543 9776 webserver.cc:460] Webserver started at http://127.9.67.254:44817/ using document root <none> and password file <none>
19:53:02.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.326429 9776 fs_manager.cc:359] Metadata directory not provided
19:53:02.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.326628 9776 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:53:02.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.334443 9776 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
19:53:02.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.338928 9795 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:53:02.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.340047 9776 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:53:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.340415 9776 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal
19:53:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5721abf73154c0abb3b3d6fe7e6f21c"
19:53:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:56 on dist-test-slave-zrlb"
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.342677 9776 fs_report.cc:383] FS layout report
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data/data
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:53:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:53:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.414460 9776 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:53:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.420702 9776 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:53:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.421255 9776 kserver.cc:163] Server-wide thread pool size limit: 3276
19:53:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.502146 9776 rpc_server.cc:307] RPC server started. Bound to: 127.9.67.254:38049
19:53:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.502310 9846 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.67.254:38049 every 8 connection(s)
19:53:02.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.505439 9776 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/data/info.pb
19:53:02.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.515086 9487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 9776
19:53:02.515 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:53:02.516 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.67.252:37603
19:53:02.516 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
19:53:02.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.516479 9847 sys_catalog.cc:263] Verifying existing consensus state
19:53:02.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 19:53:02.517391 9847 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.517809 9487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/logs
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data/info.pb
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:53:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.252:37603
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.252
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44921
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.252.keytab
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.252
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:53:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:53:02.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:53:02.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.518026 9776 master.cc:537] Master@127.9.67.254:38049 shutting down...
19:53:02.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.591982 9776 master.cc:552] Master@127.9.67.254:38049 shutdown complete.
19:53:02.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:53:02.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.855386 9849 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:53:02.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.856025 9849 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:53:02.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.856622 9849 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:53:02.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.889459 9849 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:53:02.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.889704 9849 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:53:02.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.890048 9849 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:53:02.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.890261 9849 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:53:02.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.890457 9849 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:53:02.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.890641 9849 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:53:02.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.890872 9849 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37261/jwks.json
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.930042 9849 master_runner.cc:386] Master server non-default flags:
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.252.keytab
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.252
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.252:37603
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:53:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data/info.pb
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.252
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44921
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/logs
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:53:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.931162 9849 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:53:02.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.932459 9849 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:53:02.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.950625 9857 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:04.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:04.154947 8992 debug-util.cc:398] Leaking SignalData structure 0x7b080009f4c0 after lost signal to thread 8925
19:53:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:04.156086 8992 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2900 after lost signal to thread 8995
19:53:04.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.951153 9859 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:04.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:02.951983 9849 server_base.cc:1034] running on GCE node
19:53:04.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:02.950822 9856 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:04.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.254145 9849 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:53:04.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.256917 9849 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:53:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.258309 9849 hybrid_clock.cc:648] HybridClock initialized: now 1790538784258247 us; error 80 us; skew 500 ppm
19:53:04.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.259236 9849 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:53:04.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:04 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538784, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:53:04.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.278462 9849 init.cc:377] Logged in from keytab as kudu/127.9.67.252@KRBTEST.COM (short username kudu)
19:53:04.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.285141 9849 webserver.cc:460] Webserver started at http://127.9.67.252:44921/ using document root <none> and password file <none>
19:53:04.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.286451 9849 fs_manager.cc:359] Metadata directory not provided
19:53:04.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.286741 9849 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:53:04.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.298571 9849 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.005s sys 0.004s
19:53:04.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.304706 9869 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:53:04.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.306334 9849 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
19:53:04.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.306767 9849 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal
19:53:04.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1351e3549344f58b4465974d7e78f6e"
19:53:04.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:53:00 on dist-test-slave-zrlb"
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.310747 9849 fs_report.cc:383] FS layout report
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/wal
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data/data
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:53:04.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:53:04.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:53:04.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:53:04.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:53:04.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:53:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.368695 9849 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:53:04.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.377043 9849 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:53:04.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.377609 9849 kserver.cc:163] Server-wide thread pool size limit: 3276
19:53:04.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.468896 9849 rpc_server.cc:307] RPC server started. Bound to: 127.9.67.252:37603
19:53:04.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.469013 9920 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.67.252:37603 every 8 connection(s)
19:53:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.473094 9849 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-2/data/info.pb
19:53:04.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.474221 9487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 9849
19:53:04.476 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:53:04.476 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.67.253:34749
19:53:04.476 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.478719 9487 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/logs
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data/info.pb
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:53:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:53:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:53:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:53:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:53:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:53:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:53:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:53:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:53:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:53:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:53:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.253:34749
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.253
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44605
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.253.keytab
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.253
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:53:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:53:04.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.490520 9921 sys_catalog.cc:263] Verifying existing consensus state
19:53:04.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.502705 9921 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: Bootstrap starting.
19:53:04.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.549774 9921 log.cc:824] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: Log is configured to *not* fsync() on all Append() calls
19:53:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.560551 9921 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:53:04.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.561407 9921 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: Bootstrap complete.
19:53:04.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.598762 9921 raft_consensus.cc:348] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.602872 9921 raft_consensus.cc:374] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:53:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.603246 9921 raft_consensus.cc:729] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d1351e3549344f58b4465974d7e78f6e, State: Initialized, Role: FOLLOWER
19:53:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.604461 9921 consensus_queue.cc:260] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.617051 9925 sys_catalog.cc:455] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } } }
19:53:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.618036 9925 sys_catalog.cc:458] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [sys.catalog]: This master's current role is: FOLLOWER
19:53:04.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.618970 9921 sys_catalog.cc:564] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [sys.catalog]: configured and running, proceeding with master startup.
19:53:04.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:04.634873 9875 tablet.cc:2369] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:53:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:04.661615 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:04.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:04.664687 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.662335 9849 master_runner.cc:186] Error getting master registration for 127.9.67.254:38049: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111),
19:53:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.694898 9849 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:53:04.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.718276 9925 raft_consensus.cc:482] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:53:04.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.718876 9925 raft_consensus.cc:504] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:04.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:04.726938 9873 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111)
19:53:04.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.732961 9925 leader_election.cc:288] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049), cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749)
19:53:04.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:04.747611 9870 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749): Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:53:04.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:04.750345 9873 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049): Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111)
19:53:04.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.750924 9873 leader_election.cc:302] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d1351e3549344f58b4465974d7e78f6e; no voters: a5721abf73154c0abb3b3d6fe7e6f21c, cad9fd7310c74087b8b3fc78f439d3ec
19:53:04.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:04.751901 9925 raft_consensus.cc:2726] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:53:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.003362 9923 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:53:05.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.004251 9923 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:53:05.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.005435 9923 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:53:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.070673 9923 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:53:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.071039 9923 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:53:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.071568 9923 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:53:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.072000 9923 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:53:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.072347 9923 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:53:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.072685 9923 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:53:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.072942 9923 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37261/jwks.json
19:53:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:05.148278 9923 master_runner.cc:386] Master server non-default flags:
19:53:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:53:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.67.212:40459
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.67.254:38049,127.9.67.253:34749,127.9.67.252:37603
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/krb5kdc/kudu_127.9.67.253.keytab
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.67.253
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.67.253:34749
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37261/jwks.json
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:53:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data/info.pb
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.67.253
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44605
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/logs
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:53:05.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:53:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:53:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:53:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:53:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:50:28 UTC on ccycloud.quasar-ewnlep.root.comops.site
19:53:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676277
19:53:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:53:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:05.149910 9923 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:53:05.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:05.151310 9923 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:53:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.244653 9945 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.670966 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.671685 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:06.117739 9472 debug-util.cc:398] Leaking SignalData structure 0x7b08000bec80 after lost signal to thread 9347
19:53:06.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:06.118927 9472 debug-util.cc:398] Leaking SignalData structure 0x7b08000af900 after lost signal to thread 9475
19:53:06.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:06.243912 9332 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9720 after lost signal to thread 9206
19:53:06.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:06.245150 9332 debug-util.cc:398] Leaking SignalData structure 0x7b08000afee0 after lost signal to thread 9335
19:53:06.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:06.466699 9952 raft_consensus.cc:482] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:53:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:06.467476 9952 raft_consensus.cc:504] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:06.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:06.495234 9952 leader_election.cc:288] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049), cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749)
19:53:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:06.503572 9870 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749): Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:53:06.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:06.505712 9873 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049): Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111)
19:53:06.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:06.506251 9873 leader_election.cc:302] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d1351e3549344f58b4465974d7e78f6e; no voters: a5721abf73154c0abb3b3d6fe7e6f21c, cad9fd7310c74087b8b3fc78f439d3ec
19:53:06.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:06.507093 9952 raft_consensus.cc:2726] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:53:06.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:06.676384 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:06.677093 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:07.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:07.231302 9923 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.992s user 0.576s sys 1.215s
19:53:07.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:05.248234 9946 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:07.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:07.237576 9923 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.999s user 0.576s sys 1.215s
19:53:07.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.237905 9923 server_base.cc:1034] running on GCE node
19:53:07.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:07.234751 9948 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:53:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:06.578140 9944 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 9923
19:53:07.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.240396 9923 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:53:07.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.249188 9923 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:53:07.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.250767 9923 hybrid_clock.cc:648] HybridClock initialized: now 1790538787250698 us; error 77 us; skew 500 ppm
19:53:07.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.252058 9923 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:53:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:07 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538787, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:53:07.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.273006 9923 init.cc:377] Logged in from keytab as kudu/127.9.67.253@KRBTEST.COM (short username kudu)
19:53:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.310732 9923 webserver.cc:460] Webserver started at http://127.9.67.253:44605/ using document root <none> and password file <none>
19:53:07.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.312137 9923 fs_manager.cc:359] Metadata directory not provided
19:53:07.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.312463 9923 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:53:07.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.335777 9923 fs_manager.cc:709] Time spent opening directory manager: real 0.018s user 0.006s sys 0.002s
19:53:07.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.374853 9961 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:53:07.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.376801 9923 fs_manager.cc:725] Time spent opening block manager: real 0.038s user 0.023s sys 0.003s
19:53:07.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.377434 9923 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data,/tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal
19:53:07.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cad9fd7310c74087b8b3fc78f439d3ec"
19:53:07.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 19:52:58 on dist-test-slave-zrlb"
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.381752 9923 fs_report.cc:383] FS layout report
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data/data
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:53:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:53:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:53:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:53:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:53:07.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.450417 9923 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:53:07.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.459077 9923 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:53:07.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.459980 9923 kserver.cc:163] Server-wide thread pool size limit: 3276
19:53:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:07.682318 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:07.682953 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.937500 10012 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.67.253:34749 every 8 connection(s)
19:53:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.938171 9923 rpc_server.cc:307] RPC server started. Bound to: 127.9.67.253:34749
19:53:07.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.941975 9923 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/data/info.pb
19:53:07.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.942886 9487 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu as pid 9923
19:53:07.943 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:53:07.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.966195 10013 sys_catalog.cc:263] Verifying existing consensus state
19:53:07.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 19:53:07.967298 10013 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:53:07.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:07.967743 9923 master.cc:537] Master@127.9.67.253:34749 shutting down...
19:53:08.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:08.054704 9923 master.cc:552] Master@127.9.67.253:34749 shutdown complete.
19:53:08.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskkws3i4/test-tmp/mini-kudu-cluster832836343825403290/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:53:08.064 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:08.190 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.67.252:37603(127.9.67.252:37603)] connecting to peer
19:53:08.192 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:08.275 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.9.67.252:37603(127.9.67.252:37603)] Successfully connected to peer
19:53:08.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.300 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.303 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.317 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.432 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
19:53:08.444 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.9.67.252
19:53:08.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:08.687197 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:08.688094 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:08.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:08 dist-test-slave-zrlb krb5kdc[9499](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538780, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.67.252@KRBTEST.COM
19:53:08.748 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x687eff96, L:/127.0.0.1:37920 - R:/127.9.67.252:37603] using SASL/GSSAPI
19:53:08.766 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:08.782 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:08.786 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:08.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.793 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.795 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:08.800 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:08.801 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:08.803 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.803 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.804 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.811 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:08.821 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:08.822 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:08.824 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.824 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.825 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.830 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.830 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.832 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:08.841 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:08.842 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:08.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:08.854125 10019 raft_consensus.cc:482] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:53:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:08.854724 10019 raft_consensus.cc:504] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:08.870 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:08.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:08.874449 10019 leader_election.cc:288] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049), cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749)
19:53:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:08.880273 9870 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749): Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:53:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:08.881026 9873 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049): Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111)
19:53:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:08.881346 9873 leader_election.cc:302] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d1351e3549344f58b4465974d7e78f6e; no voters: a5721abf73154c0abb3b3d6fe7e6f21c, cad9fd7310c74087b8b3fc78f439d3ec
19:53:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:08.882061 10019 raft_consensus.cc:2726] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:53:08.901 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:08.902 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:08.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.908 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:08.981 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:08.984 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.984 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:08.985 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:08.987 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.987 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.988 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:08.990 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:09.121 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:09.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:09.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:09.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:09.127 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:09.134 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:09.134 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:09.135 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:09.139 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:09.361 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:09.363 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:09.364 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:09.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:09.367 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:09.368 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:09.368 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:09.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:09.369 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:09.481 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:09.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:09.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:09.488 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:09.483 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:09.493 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:09.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:09.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:09.494 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:09.691421 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:09.691684 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:09.697708 9849 master_runner.cc:186] Error getting master registration for 127.9.67.254:38049: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111),
19:53:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:09.698048 9849 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:53:10.401 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:10.403 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:10.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:10.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:10.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:10.406 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:10.406 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:10.406 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:10.407 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:10.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:10.443418 10027 raft_consensus.cc:482] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:53:10.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:10.443846 10027 raft_consensus.cc:504] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:10.447242 10027 leader_election.cc:288] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049), cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749)
19:53:10.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:10.449491 9873 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111) [suppressed 5 similar messages]
19:53:10.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:10.452473 9873 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049): Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111)
19:53:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:10.453073 9870 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749): Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:53:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:10.453457 9870 leader_election.cc:302] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d1351e3549344f58b4465974d7e78f6e; no voters: a5721abf73154c0abb3b3d6fe7e6f21c, cad9fd7310c74087b8b3fc78f439d3ec
19:53:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:10.454093 10027 raft_consensus.cc:2726] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:53:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:10.694801 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:10.695047 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:11.221 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:11.223 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:11.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:11.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:11.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:11.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:11.226 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:11.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:11.227 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:11.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:11.697942 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:11.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:11.698213 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:12.521 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:12.522 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:12.524 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:12.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:12.525 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:12.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:12.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:12.526 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:12.527 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:12.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:12.701706 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:12.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:12.702071 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:13.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:13.386555 9472 debug-util.cc:398] Leaking SignalData structure 0x7b08000bc6e0 after lost signal to thread 9347
19:53:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:13.387421 9472 debug-util.cc:398] Leaking SignalData structure 0x7b08000aff80 after lost signal to thread 9475
19:53:13.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:13.705983 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:13.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:13.706323 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:14.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:14 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538794, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:53:14.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:14.282968 9865 init.cc:295] Successfully reacquired a new kerberos TGT
19:53:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:14.701802 9849 master_runner.cc:186] Error getting master registration for 127.9.67.254:38049: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111),
19:53:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:14.702196 9849 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:53:14.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:14.709080 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:14.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:14.709329 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:14.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:14.945381 10037 raft_consensus.cc:482] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:53:14.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:14.945891 10037 raft_consensus.cc:504] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:14.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:14.947505 10037 leader_election.cc:288] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049), cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749)
19:53:14.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:14.951481 9873 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049): Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111)
19:53:14.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:14.952011 9870 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749): Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:53:14.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:14.952297 9870 leader_election.cc:302] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d1351e3549344f58b4465974d7e78f6e; no voters: a5721abf73154c0abb3b3d6fe7e6f21c, cad9fd7310c74087b8b3fc78f439d3ec
19:53:14.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:14.952845 10037 raft_consensus.cc:2726] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:53:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:15.712256 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:15.712505 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:15.861 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:15.862 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:15.864 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:15.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:15.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:15.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:15.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:15.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:15.867 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:16.715199 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:16.715448 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:17.718173 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:17.718432 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:17.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:17.784353 10041 raft_consensus.cc:482] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:53:17.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:17.784796 10041 raft_consensus.cc:504] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:17.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:17.786592 10041 leader_election.cc:288] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049), cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749)
19:53:17.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:17.788596 9873 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111) [suppressed 3 similar messages]
19:53:17.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:17.791235 9870 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749): Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:53:17.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:17.791687 9873 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049): Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111)
19:53:17.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:17.792062 9873 leader_election.cc:302] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d1351e3549344f58b4465974d7e78f6e; no voters: a5721abf73154c0abb3b3d6fe7e6f21c, cad9fd7310c74087b8b3fc78f439d3ec
19:53:17.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:17.792578 10041 raft_consensus.cc:2726] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:53:18.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:18.721306 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:18.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:18.721632 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:19.581 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:19.583 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:19.584 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:19.584 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:19.585 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:19.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:19.585 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:19.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:19.586 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:19.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:19.704851 9849 master_runner.cc:186] Error getting master registration for 127.9.67.254:38049: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111),
19:53:19.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:19.705216 9849 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:53:19.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:19.724571 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:19.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:19.724882 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:19.961 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:19.963 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:19.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:19.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:19.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:19.966 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:19.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:19.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:19.967 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:20.727600 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:20.727936 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:21.481 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:21.483 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:21.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:21.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:21.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:21.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:21.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:21.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:21.487 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:21.730914 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:21.731177 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:22.733853 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:22.734112 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:23.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:23.701330 9332 debug-util.cc:398] Leaking SignalData structure 0x7b08000af7c0 after lost signal to thread 9206
19:53:23.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:23.702106 9332 debug-util.cc:398] Leaking SignalData structure 0x7b08000afb80 after lost signal to thread 9335
19:53:23.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:23.738207 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:23.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:23.738530 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:24.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:24 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538804, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:53:24.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:24.285593 9865 init.cc:295] Successfully reacquired a new kerberos TGT
19:53:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:24.708419 9849 master_runner.cc:186] Error getting master registration for 127.9.67.254:38049: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111),
19:53:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:24.708835 9849 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:53:24.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:24.741271 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:24.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:24.741580 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:25.334935 10055 raft_consensus.cc:482] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:53:25.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:25.335538 10055 raft_consensus.cc:504] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5721abf73154c0abb3b3d6fe7e6f21c" member_type: VOTER last_known_addr { host: "127.9.67.254" port: 38049 } } peers { permanent_uuid: "cad9fd7310c74087b8b3fc78f439d3ec" member_type: VOTER last_known_addr { host: "127.9.67.253" port: 34749 } } peers { permanent_uuid: "d1351e3549344f58b4465974d7e78f6e" member_type: VOTER last_known_addr { host: "127.9.67.252" port: 37603 } }
19:53:25.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:25.337801 10055 leader_election.cc:288] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049), cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749)
19:53:25.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:25.340250 9873 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111) [suppressed 1 similar messages]
19:53:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:25.342903 9873 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5721abf73154c0abb3b3d6fe7e6f21c (127.9.67.254:38049): Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111)
19:53:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:25.343520 9870 leader_election.cc:334] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cad9fd7310c74087b8b3fc78f439d3ec (127.9.67.253:34749): Network error: Client connection negotiation failed: client connection to 127.9.67.253:34749: connect: Connection refused (error 111)
19:53:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:25.343863 9870 leader_election.cc:302] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d1351e3549344f58b4465974d7e78f6e; no voters: a5721abf73154c0abb3b3d6fe7e6f21c, cad9fd7310c74087b8b3fc78f439d3ec
19:53:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:25.344483 10055 raft_consensus.cc:2726] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:53:25.381 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:25.382 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:25.384 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:25.384 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:25.384 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:25.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:25.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:25.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:25.388 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:25.744427 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:25.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:25.744683 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:26.747404 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:26.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:26.747661 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:27.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:27.750394 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:27.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:27.750655 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:28.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:28.753269 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:28.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:28.753508 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:29.081 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:29.083 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:29.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:29.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:29.085 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:29.085 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:29.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:29.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:29.087 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:29.218533 9191 debug-util.cc:398] Leaking SignalData structure 0x7b08000bc280 after lost signal to thread 9064
19:53:29.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:29.219290 9191 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcc80 after lost signal to thread 9194
19:53:29.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:29.711866 9849 master_runner.cc:186] Error getting master registration for 127.9.67.254:38049: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111),
19:53:29.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:29.712237 9849 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:53:29.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:29.756337 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:29.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:29.756624 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:30.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:30.759351 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:30.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:30.759649 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:31.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:31.762575 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:31.762893 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:31.941 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:31.943 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:31.944 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:31.944 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:31.944 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:31.944 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:31.944 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:31.944 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:31.947 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:32.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:32.765676 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:32.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:32.765978 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:33.681 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:33.682 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:33.683 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:33.683 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:33.684 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:33.684 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:33.684 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:33.684 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:33.687 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:33.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:33.769265 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:33.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:33.769840 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:34.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 19:53:34 dist-test-slave-zrlb krb5kdc[9499](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790538814, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.67.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:53:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:34.288139 9865 init.cc:295] Successfully reacquired a new kerberos TGT
19:53:34.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:34.715269 9849 master_runner.cc:186] Error getting master registration for 127.9.67.254:38049: Network error: Client connection negotiation failed: client connection to 127.9.67.254:38049: connect: Connection refused (error 111),
19:53:34.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:34.715654 9849 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:53:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:34.772709 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:34.772980 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:35.775425 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:35.775662 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:36.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:36.778074 9936 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1351e3549344f58b4465974d7e78f6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:53:36.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:36.778319 9936 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:53:37.721 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.254:38049(127.9.67.254:38049)] connecting to peer
19:53:37.722 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.67.253:34749(127.9.67.253:34749)] connecting to peer
19:53:37.723 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:37.723 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:37.723 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.67.253:34749(127.9.67.253:34749)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:37.723 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.254:38049(127.9.67.254:38049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:37.723 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.254:38049: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049
19:53:37.723 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.67.253:34749: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
19:53:37.725 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749; will retry
19:53:37.730 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29778), Traces: [0ms] refreshing cache from master, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [337ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [346ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [797ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [802ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [816ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [821ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [824ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [828ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [829ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [838ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [843ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [844ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [847ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [859ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [864ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [865ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [867ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [875ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [881ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [886ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [905ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [942ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [944ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [945ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1020ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1025ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1027ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1155ms] refreshing cache from master, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1160ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1169ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1173ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1174ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1395ms] refreshing cache from master, [1396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1405ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1515ms] refreshing cache from master, [1516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1520ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1524ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1528ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1529ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [2436ms] refreshing cache from master, [2436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [2437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [2438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [2440ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [2441ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [2442ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [2443ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [3256ms] refreshing cache from master, [3256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [3258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [3260ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [3261ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [3263ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [4555ms] refreshing cache from master, [4555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [4557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [4557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [4560ms] too many traces: truncated at 100 traces, deferred=Deferred@675493456(state=PENDING, result=null, callback=(continuation of Deferred@353525075 after retry RPC after error@286040286) -> (continuation of Deferred@2139101479 after retry RPC after error@1321226545) -> (continuation of Deferred@1237284223 after retry RPC after error@1027679942) -> (continuation of Deferred@1115117928 after retry RPC after error@250794446) -> (continuation of Deferred@1012156898 after retry RPC after error@144129976) -> (continuation of Deferred@1884527579 after retry RPC after error@1756842174) -> (continuation of Deferred@1991382803 after retry RPC after error@1198103682) -> (continuation of Deferred@721967863 after retry RPC after error@2041054608) -> (continuation of Deferred@532979513 after retry RPC after error@339905999) -> (continuation of Deferred@2104795730 after retry RPC after error@1145679081) -> (continuation of Deferred@2144329230 after retry RPC after error@2137196749) -> (continuation of Deferred@668812097 after retry RPC after error@779783607) -> (continuation of Deferred@792988120 after retry RPC after error@1216764317) -> (continuation of Deferred@93117575 after retry RPC after error@2145513140) -> (continuation of Deferred@1885614681 after retry RPC after error@173328249) -> (continuation of Deferred@1776241232 after retry RPC after error@431721873) -> (continuation of Deferred@1209827845 after retry RPC after error@1692396400) -> (continuation of Deferred@2000793713 after retry RPC after error@844332045) -> (continuation of Deferred@519298280 after retry RPC after error@90972205) -> (continuation of Deferred@2136263369 after retry RPC after error@1447977530) -> (continuation of Deferred@1448800391 after retry RPC after error@852741929), errback=(continuation of Deferred@353525075 after retry RPC after error@286040286) -> (continuation of Deferred@2139101479 after retry RPC after error@1321226545) -> (continuation of Deferred@1237284223 after retry RPC after error@1027679942) -> (continuation of Deferred@1115117928 after retry RPC after error@250794446) -> (continuation of Deferred@1012156898 after retry RPC after error@144129976) -> (continuation of Deferred@1884527579 after retry RPC after error@1756842174) -> (continuation of Deferred@1991382803 after retry RPC after error@1198103682) -> (continuation of Deferred@721967863 after retry RPC after error@2041054608) -> (continuation of Deferred@532979513 after retry RPC after error@339905999) -> (continuation of Deferred@2104795730 after retry RPC after error@1145679081) -> (continuation of Deferred@2144329230 after retry RPC after error@2137196749) -> (continuation of Deferred@668812097 after retry RPC after error@779783607) -> (continuation of Deferred@792988120 after retry RPC after error@1216764317) -> (continuation of Deferred@93117575 after retry RPC after error@2145513140) -> (continuation of Deferred@1885614681 after retry RPC after error@173328249) -> (continuation of Deferred@1776241232 after retry RPC after error@431721873) -> (continuation of Deferred@1209827845 after retry RPC after error@1692396400) -> (continuation of Deferred@2000793713 after retry RPC after error@844332045) -> (continuation of Deferred@519298280 after retry RPC after error@90972205) -> (continuation of Deferred@2136263369 after retry RPC after error@1447977530) -> (continuation of Deferred@1448800391 after retry RPC after error@852741929)))
19:53:37.735 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.67.253:34749(127.9.67.253:34749)] disconnecting while in state TERMINATED
19:53:37.736 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.67.252:37603(127.9.67.252:37603)] disconnecting while in state READY
19:53:37.736 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.67.254:38049(127.9.67.254:38049)] disconnecting while in state TERMINATED
19:53:37.741 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:53:37.743 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:53:37.744 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.9.67.252:37603(127.9.67.252:37603)] handling channelInactive
19:53:37.744 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.67.252:37603(127.9.67.252:37603)] cleaning up while in state READY due to: connection closed
19:53:37.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:37.749652 9487 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu with pid 9849
19:53:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:38.640527 8839 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9280 after lost signal to thread 8776
19:53:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:38.641244 8839 debug-util.cc:398] Leaking SignalData structure 0x7b08000b93a0 after lost signal to thread 8842
19:53:38.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T19:53:38Z chronyd exiting
19:53:39.978 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:53:39.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:39.980350 8763 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu with pid 9043
19:53:40.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:40.006567 8763 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu with pid 9199
19:53:40.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:40.033988 8763 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu with pid 9346
19:53:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:40.061830 8763 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu with pid 8775
19:53:40.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 19:53:40.094705 8947 connection.cc:535] server connection from 127.0.0.1:60200 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
19:53:40.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:40.095532 8763 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu with pid 8845
19:53:40.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 19:53:40.122318 8763 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskkws3i4/build/tsan/bin/kudu with pid 8920
19:53:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T19:53:41Z chronyd exiting
19:53:42.307 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29778), Traces: [0ms] refreshing cache from master, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [337ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [346ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [797ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [802ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [816ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [821ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [824ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [828ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [829ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [838ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [843ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [844ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [847ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [859ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [864ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [865ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [867ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [875ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [881ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [886ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [905ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [942ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [944ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [945ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1020ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1025ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1027ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1155ms] refreshing cache from master, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1160ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1169ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1173ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1174ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1395ms] refreshing cache from master, [1396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1405ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1515ms] refreshing cache from master, [1516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1520ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1524ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1528ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1529ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [2436ms] refreshing cache from master, [2436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [2437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [2438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [2440ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [2441ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [2442ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [2443ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [3256ms] refreshing cache from master, [3256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [3258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [3260ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [3261ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [3263ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [4555ms] refreshing cache from master, [4555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [4557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [4557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [4560ms] too many traces: truncated at 100 traces, deferred=Deferred@675493456(state=PENDING, result=null, callback=(continuation of Deferred@353525075 after retry RPC after error@286040286) -> (continuation of Deferred@2139101479 after retry RPC after error@1321226545) -> (continuation of Deferred@1237284223 after retry RPC after error@1027679942) -> (continuation of Deferred@1115117928 after retry RPC after error@250794446) -> (continuation of Deferred@1012156898 after retry RPC after error@144129976) -> (continuation of Deferred@1884527579 after retry RPC after error@1756842174) -> (continuation of Deferred@1991382803 after retry RPC after error@1198103682) -> (continuation of Deferred@721967863 after retry RPC after error@2041054608) -> (continuation of Deferred@532979513 after retry RPC after error@339905999) -> (continuation of Deferred@2104795730 after retry RPC after error@1145679081) -> (continuation of Deferred@2144329230 after retry RPC after error@2137196749) -> (continuation of Deferred@668812097 after retry RPC after error@779783607) -> (continuation of Deferred@792988120 after retry RPC after error@1216764317) -> (continuation of Deferred@93117575 after retry RPC after error@2145513140) -> (continuation of Deferred@1885614681 after retry RPC after error@173328249) -> (continuation of Deferred@1776241232 after retry RPC after error@431721873) -> (continuation of Deferred@1209827845 after retry RPC after error@1692396400) -> (continuation of Deferred@2000793713 after retry RPC after error@844332045) -> (continuation of Deferred@519298280 after retry RPC after error@90972205) -> (continuation of Deferred@2136263369 after retry RPC after error@1447977530) -> (continuation of Deferred@1448800391 after retry RPC after error@852741929), errback=(continuation of Deferred@353525075 after retry RPC after error@286040286) -> (continuation of Deferred@2139101479 after retry RPC after error@1321226545) -> (continuation of Deferred@1237284223 after retry RPC after error@1027679942) -> (continuation of Deferred@1115117928 after retry RPC after error@250794446) -> (continuation of Deferred@1012156898 after retry RPC after error@144129976) -> (continuation of Deferred@1884527579 after retry RPC after error@1756842174) -> (continuation of Deferred@1991382803 after retry RPC after error@1198103682) -> (continuation of Deferred@721967863 after retry RPC after error@2041054608) -> (continuation of Deferred@532979513 after retry RPC after error@339905999) -> (continuation of Deferred@2104795730 after retry RPC after error@1145679081) -> (continuation of Deferred@2144329230 after retry RPC after error@2137196749) -> (continuation of Deferred@668812097 after retry RPC after error@779783607) -> (continuation of Deferred@792988120 after retry RPC after error@1216764317) -> (continuation of Deferred@93117575 after retry RPC after error@2145513140) -> (continuation of Deferred@1885614681 after retry RPC after error@173328249) -> (continuation of Deferred@1776241232 after retry RPC after error@431721873) -> (continuation of Deferred@1209827845 after retry RPC after error@1692396400) -> (continuation of Deferred@2000793713 after retry RPC after error@844332045) -> (continuation of Deferred@519298280 after retry RPC after error@90972205) -> (continuation of Deferred@2136263369 after retry RPC after error@1447977530) -> (continuation of Deferred@1448800391 after retry RPC after error@852741929)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.67.253:34749
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29778), Traces: [0ms] refreshing cache from master, [87ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [337ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [346ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [797ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [802ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [816ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [821ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [824ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [828ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [829ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [838ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [843ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [844ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [847ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [859ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [864ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [865ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [867ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [875ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [881ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [886ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [905ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [942ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [944ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [945ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1020ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1025ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1027ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1155ms] refreshing cache from master, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1160ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1169ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1173ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1174ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1395ms] refreshing cache from master, [1396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1405ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1515ms] refreshing cache from master, [1516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [1518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [1520ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [1524ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [1528ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [1529ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [2436ms] refreshing cache from master, [2436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [2437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [2438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [2440ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [2441ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [2442ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [2443ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [3256ms] refreshing cache from master, [3256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [3258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [3260ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.253:34749: Network error: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749, [3261ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.254:38049: Network error: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.9.67.252:37603: OK, [3263ms] delaying RPC due to: Service unavailable: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049, [4555ms] refreshing cache from master, [4555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.254:38049, [4557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.252:37603, [4557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.67.253:34749, [4560ms] too many traces: truncated at 100 traces, deferred=Deferred@675493456(state=PENDING, result=null, callback=(continuation of Deferred@353525075 after retry RPC after error@286040286) -> (continuation of Deferred@2139101479 after retry RPC after error@1321226545) -> (continuation of Deferred@1237284223 after retry RPC after error@1027679942) -> (continuation of Deferred@1115117928 after retry RPC after error@250794446) -> (continuation of Deferred@1012156898 after retry RPC after error@144129976) -> (continuation of Deferred@1884527579 after retry RPC after error@1756842174) -> (continuation of Deferred@1991382803 after retry RPC after error@1198103682) -> (continuation of Deferred@721967863 after retry RPC after error@2041054608) -> (continuation of Deferred@532979513 after retry RPC after error@339905999) -> (continuation of Deferred@2104795730 after retry RPC after error@1145679081) -> (continuation of Deferred@2144329230 after retry RPC after error@2137196749) -> (continuation of Deferred@668812097 after retry RPC after error@779783607) -> (continuation of Deferred@792988120 after retry RPC after error@1216764317) -> (continuation of Deferred@93117575 after retry RPC after error@2145513140) -> (continuation of Deferred@1885614681 after retry RPC after error@173328249) -> (continuation of Deferred@1776241232 after retry RPC after error@431721873) -> (continuation of Deferred@1209827845 after retry RPC after error@1692396400) -> (continuation of Deferred@2000793713 after retry RPC after error@844332045) -> (continuation of Deferred@519298280 after retry RPC after error@90972205) -> (continuation of Deferred@2136263369 after retry RPC after error@1447977530) -> (continuation of Deferred@1448800391 after retry RPC after error@852741929), errback=(continuation of Deferred@353525075 after retry RPC after error@286040286) -> (continuation of Deferred@2139101479 after retry RPC after error@1321226545) -> (continuation of Deferred@1237284223 after retry RPC after error@1027679942) -> (continuation of Deferred@1115117928 after retry RPC after error@250794446) -> (continuation of Deferred@1012156898 after retry RPC after error@144129976) -> (continuation of Deferred@1884527579 after retry RPC after error@1756842174) -> (continuation of Deferred@1991382803 after retry RPC after error@1198103682) -> (continuation of Deferred@721967863 after retry RPC after error@2041054608) -> (continuation of Deferred@532979513 after retry RPC after error@339905999) -> (continuation of Deferred@2104795730 after retry RPC after error@1145679081) -> (continuation of Deferred@2144329230 after retry RPC after error@2137196749) -> (continuation of Deferred@668812097 after retry RPC after error@779783607) -> (continuation of Deferred@792988120 after retry RPC after error@1216764317) -> (continuation of Deferred@93117575 after retry RPC after error@2145513140) -> (continuation of Deferred@1885614681 after retry RPC after error@173328249) -> (continuation of Deferred@1776241232 after retry RPC after error@431721873) -> (continuation of Deferred@1209827845 after retry RPC after error@1692396400) -> (continuation of Deferred@2000793713 after retry RPC after error@844332045) -> (continuation of Deferred@519298280 after retry RPC after error@90972205) -> (continuation of Deferred@2136263369 after retry RPC after error@1447977530) -> (continuation of Deferred@1448800391 after retry RPC after error@852741929)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.67.254:38049,127.9.67.252:37603,127.9.67.253:34749) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.254:38049(127.9.67.254:38049): Connection refused: /127.9.67.254:38049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.67.253:34749(127.9.67.253:34749): Connection refused: /127.9.67.253:34749
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.67.253:34749
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)