19:44:33.275 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
19:44:33.290 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskBYv0IG/test-tmp
19:44:33.293 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:44:33.297 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskBYv0IG/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:44:33.613 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
19:44:33.756 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:44:33.757 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:44:33.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:33.762892 28720 mini_oidc.cc:152] Starting JWKS server
19:44:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:33.767908 28720 webserver.cc:460] Webserver started at http://127.0.0.1:42861/ using document root <none> and password file <none>
19:44:33.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:33.768728 28720 mini_oidc.cc:189] Starting OIDC Discovery server
19:44:33.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:33.771229 28720 webserver.cc:460] Webserver started at http://127.0.0.1:44797/ using document root <none> and password file <none>
19:44:33.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T19:44:33Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:44:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T19:44:33Z Disabled control of system clock
19:44:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:33.827503 28720 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/wal
19:44:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/data
19:44:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:44:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:44:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:44:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/logs
19:44:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/data/info.pb
19:44:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:44:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:44:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:44:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:44:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:44:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:44:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.62:41295
19:44:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.62
19:44:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:33.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:33.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:33.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:33.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:33.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:33.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:33.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:33.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42861/jwks.json with env {}
19:44:34.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.158587 28732 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:44:34.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.159178 28732 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:44:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.159821 28732 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:44:34.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.191238 28732 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:44:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.191537 28732 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:44:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.191725 28732 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:44:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.191900 28732 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:44:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.192209 28732 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:44:34.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.192382 28732 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42861/jwks.json
19:44:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:34.228864 28732 master_runner.cc:386] Master server non-default flags:
19:44:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/data
19:44:34.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/wal
19:44:34.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.62:41295
19:44:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:44:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42861/jwks.json
19:44:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/data/info.pb
19:44:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.62
19:44:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:44:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:44:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:44:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/logs
19:44:34.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:44:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:44:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:44:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:44:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:44:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:44:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:44:34.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:44:34.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:44:34.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:34.230597 28732 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:44:34.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:34.232625 28732 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:44:35.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.254906 28738 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.254850 28739 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:34.254947 28741 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:34.256178 28732 server_base.cc:1034] running on GCE node
19:44:35.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.455878 28732 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:44:35.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.458755 28732 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:44:35.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.460136 28732 hybrid_clock.cc:648] HybridClock initialized: now 1786045475460101 us; error 125 us; skew 500 ppm
19:44:35.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.461062 28732 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:44:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.470999 28732 webserver.cc:460] Webserver started at http://127.28.12.62:37097/ using document root <none> and password file <none>
19:44:35.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.471958 28732 fs_manager.cc:359] Metadata directory not provided
19:44:35.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.472152 28732 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:44:35.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.472546 28732 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:44:35.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.477886 28732 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/data/instance:
19:44:35.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f710f73642f14b42812630199af9017d"
19:44:35.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:35 on dist-test-slave-vxj2"
19:44:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.479187 28732 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/wal/instance:
19:44:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f710f73642f14b42812630199af9017d"
19:44:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:35 on dist-test-slave-vxj2"
19:44:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.487496 28732 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
19:44:35.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.493296 28748 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:44:35.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.494598 28732 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
19:44:35.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.494930 28732 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/wal
19:44:35.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f710f73642f14b42812630199af9017d"
19:44:35.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:35 on dist-test-slave-vxj2"
19:44:35.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.495349 28732 fs_report.cc:383] FS layout report
19:44:35.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:44:35.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/wal
19:44:35.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/wal
19:44:35.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/data/data
19:44:35.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:44:35.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:44:35.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:44:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:44:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:44:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:44:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:44:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:44:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:44:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:44:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:44:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.538691 28732 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:44:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.540293 28732 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:44:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.540768 28732 kserver.cc:163] Server-wide thread pool size limit: 3276
19:44:35.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.618096 28732 rpc_server.cc:307] RPC server started. Bound to: 127.28.12.62:41295
19:44:35.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.618221 28800 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.12.62:41295 every 8 connection(s)
19:44:35.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.620955 28732 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/data/info.pb
19:44:35.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.626873 28801 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:44:35.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.627748 28720 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 28732
19:44:35.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.628279 28720 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-0/wal/instance
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.633569 28720 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/wal
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/data
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/logs
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/data/info.pb
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:44:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:44:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:44:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:44:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:44:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:44:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.61:43273
19:44:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.61
19:44:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:35.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42861/jwks.json with env {}
19:44:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.643051 28801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } has no permanent_uuid. Determining permanent_uuid...
19:44:35.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.673628 28801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } has no permanent_uuid. Determining permanent_uuid...
19:44:35.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.676103 28751 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.12.61:43273: connect: Connection refused (error 111)
19:44:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.680707 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.61:43273: Network error: Client connection negotiation failed: client connection to 127.28.12.61:43273: connect: Connection refused (error 111)
19:44:35.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.730266 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } attempt: 1
19:44:35.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.734444 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.61:43273: Network error: Client connection negotiation failed: client connection to 127.28.12.61:43273: connect: Connection refused (error 111)
19:44:35.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.802851 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } attempt: 2
19:44:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.807201 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.61:43273: Network error: Client connection negotiation failed: client connection to 127.28.12.61:43273: connect: Connection refused (error 111)
19:44:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:35.898698 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } attempt: 3
19:44:35.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.903082 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.61:43273: Network error: Client connection negotiation failed: client connection to 127.28.12.61:43273: connect: Connection refused (error 111)
19:44:35.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.957419 28803 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:44:35.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.958074 28803 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:44:35.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.958639 28803 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:44:35.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.990705 28803 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:44:35.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.991020 28803 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:44:35.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.991216 28803 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:44:35.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.991573 28803 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:44:35.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.991888 28803 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:44:35.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:35.992105 28803 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42861/jwks.json
19:44:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:36.028584 28803 master_runner.cc:386] Master server non-default flags:
19:44:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/data
19:44:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/wal
19:44:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.61:43273
19:44:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:44:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42861/jwks.json
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/data/info.pb
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.61
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/logs
19:44:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:44:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:44:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:44:36.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:44:36.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:44:36.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:44:36.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:44:36.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:44:36.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:44:36.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:36.029867 28803 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:44:36.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:36.031546 28803 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:44:36.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:36.046514 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } attempt: 4
19:44:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:36.049626 28813 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:36.052485 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.61:43273: Network error: Client connection negotiation failed: client connection to 127.28.12.61:43273: connect: Connection refused (error 111)
19:44:36.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:36.351948 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } attempt: 5
19:44:36.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:36.357128 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.61:43273: Network error: Client connection negotiation failed: client connection to 127.28.12.61:43273: connect: Connection refused (error 111)
19:44:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:36.904645 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } attempt: 6
19:44:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:36.914083 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.61:43273: Network error: Client connection negotiation failed: client connection to 127.28.12.61:43273: connect: Connection refused (error 111)
19:44:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:36.050855 28812 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:36.052402 28803 server_base.cc:1034] running on GCE node
19:44:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:36.051110 28815 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:37.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.289026 28803 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:44:37.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.292286 28803 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:44:37.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.293711 28803 hybrid_clock.cc:648] HybridClock initialized: now 1786045477293682 us; error 48 us; skew 500 ppm
19:44:37.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.294829 28803 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:44:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.301513 28803 webserver.cc:460] Webserver started at http://127.28.12.61:45433/ using document root <none> and password file <none>
19:44:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.302479 28803 fs_manager.cc:359] Metadata directory not provided
19:44:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.302663 28803 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:44:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.303061 28803 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:44:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.308019 28803 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/data/instance:
19:44:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc7f813133994aaab80e1bbd5beea8df"
19:44:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:37 on dist-test-slave-vxj2"
19:44:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.310370 28803 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/wal/instance:
19:44:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc7f813133994aaab80e1bbd5beea8df"
19:44:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:37 on dist-test-slave-vxj2"
19:44:37.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.320856 28803 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.010s sys 0.000s
19:44:37.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.327299 28826 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:44:37.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.328454 28803 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
19:44:37.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.328826 28803 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/wal
19:44:37.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc7f813133994aaab80e1bbd5beea8df"
19:44:37.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:37 on dist-test-slave-vxj2"
19:44:37.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.329327 28803 fs_report.cc:383] FS layout report
19:44:37.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:44:37.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/wal
19:44:37.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/wal
19:44:37.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/data/data
19:44:37.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:44:37.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:44:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:44:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:44:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:44:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:44:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:44:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:44:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:44:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:44:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:44:37.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.369552 28803 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:44:37.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.371220 28803 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:44:37.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.371762 28803 kserver.cc:163] Server-wide thread pool size limit: 3276
19:44:37.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.449105 28803 rpc_server.cc:307] RPC server started. Bound to: 127.28.12.61:43273
19:44:37.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.449317 28878 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.12.61:43273 every 8 connection(s)
19:44:37.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.452065 28803 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/data/info.pb
19:44:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.454474 28720 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 28803
19:44:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.454983 28720 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-1/wal/instance
19:44:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.458048 28720 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/wal
19:44:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/data
19:44:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:44:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:44:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/logs
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/data/info.pb
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.60:32861
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.60
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42861/jwks.json with env {}
19:44:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.458765 28879 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:44:37.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.472012 28879 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } has no permanent_uuid. Determining permanent_uuid...
19:44:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.496688 28879 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } has no permanent_uuid. Determining permanent_uuid...
19:44:37.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.513597 28879 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } has no permanent_uuid. Determining permanent_uuid...
19:44:37.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.515974 28827 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:37.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.520502 28879 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:37.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.570119 28879 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 1
19:44:37.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.575085 28879 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:37.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.643476 28879 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 2
19:44:37.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.647815 28879 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.739223 28879 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 3
19:44:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.743134 28879 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.782366 28881 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:44:37.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.783113 28881 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:44:37.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.783849 28881 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:44:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.816015 28881 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:44:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.816417 28881 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:44:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.816746 28881 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:44:37.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.817098 28881 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:44:37.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.817502 28881 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:44:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.817790 28881 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42861/jwks.json
19:44:37.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.856072 28881 master_runner.cc:386] Master server non-default flags:
19:44:37.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:37.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/data
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/wal
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.60:32861
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42861/jwks.json
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/data/info.pb
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.60
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/logs
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:44:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:44:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:44:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:44:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:44:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:44:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:44:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:44:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:44:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.857735 28881 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:44:37.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.859975 28881 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:44:37.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.879125 28892 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:37.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.886534 28879 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 4
19:44:37.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.890504 28879 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:37.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.974803 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } attempt: 7
19:44:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:37.989094 28801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } has no permanent_uuid. Determining permanent_uuid...
19:44:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.994773 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:38.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:38.053323 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 1
19:44:38.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:38.057909 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:38.139348 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 2
19:44:38.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:38.144016 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:38.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:38.189986 28879 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 5
19:44:38.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:38.195520 28879 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:38.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:38.229482 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 3
19:44:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:38.234220 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:38.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:38.374668 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 4
19:44:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:38.379405 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:38.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:38.663046 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 5
19:44:38.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:38.668953 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:38.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:38.742995 28879 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 6
19:44:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:38.751442 28879 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:39.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.879273 28894 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:39.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:37.879266 28891 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:39.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:39.095487 28893 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:44:39.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.095705 28881 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:44:39.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.099432 28881 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:44:39.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.102135 28881 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:44:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.103528 28881 hybrid_clock.cc:648] HybridClock initialized: now 1786045479103501 us; error 50 us; skew 500 ppm
19:44:39.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.104547 28881 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:44:39.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.112823 28881 webserver.cc:460] Webserver started at http://127.28.12.60:36435/ using document root <none> and password file <none>
19:44:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.114032 28881 fs_manager.cc:359] Metadata directory not provided
19:44:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.114272 28881 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:44:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.114806 28881 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:44:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.120741 28881 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/data/instance:
19:44:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aafc485e9df84955baa520eec202428d"
19:44:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:39 on dist-test-slave-vxj2"
19:44:39.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.122124 28881 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/wal/instance:
19:44:39.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aafc485e9df84955baa520eec202428d"
19:44:39.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:39 on dist-test-slave-vxj2"
19:44:39.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.132460 28881 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.001s sys 0.008s
19:44:39.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.139256 28906 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:44:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.140579 28881 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
19:44:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.140895 28881 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/wal
19:44:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aafc485e9df84955baa520eec202428d"
19:44:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:39 on dist-test-slave-vxj2"
19:44:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.141285 28881 fs_report.cc:383] FS layout report
19:44:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:44:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/wal
19:44:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/wal
19:44:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/data/data
19:44:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:44:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:44:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:44:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:44:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:44:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:44:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:44:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:44:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:44:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:44:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:44:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.184716 28881 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:44:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.186414 28881 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:44:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.186842 28881 kserver.cc:163] Server-wide thread pool size limit: 3276
19:44:39.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.221474 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 6
19:44:39.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:39.229442 28801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.12.60:32861: Network error: Client connection negotiation failed: client connection to 127.28.12.60:32861: connect: Connection refused (error 111)
19:44:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.263942 28881 rpc_server.cc:307] RPC server started. Bound to: 127.28.12.60:32861
19:44:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.264045 28960 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.12.60:32861 every 8 connection(s)
19:44:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.267741 28881 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/data/info.pb
19:44:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.268360 28720 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 28881
19:44:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.268792 28720 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/master-2/wal/instance
19:44:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.275684 28961 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:44:39.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.297886 28961 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } has no permanent_uuid. Determining permanent_uuid...
19:44:39.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.326107 28961 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } has no permanent_uuid. Determining permanent_uuid...
19:44:39.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.339537 28961 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } has no permanent_uuid. Determining permanent_uuid...
19:44:39.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.358958 28961 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d: Bootstrap starting.
19:44:39.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.365460 28961 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d: Neither blocks nor log segments found. Creating new log.
19:44:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.367619 28961 log.cc:824] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d: Log is configured to *not* fsync() on all Append() calls
19:44:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.373543 28961 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d: No bootstrap required, opened a new log
19:44:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.400192 28961 raft_consensus.cc:348] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } }
19:44:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.401068 28961 raft_consensus.cc:374] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:44:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.401371 28961 raft_consensus.cc:729] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aafc485e9df84955baa520eec202428d, State: Initialized, Role: FOLLOWER
19:44:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.402153 28961 consensus_queue.cc:260] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } }
19:44:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.404711 28970 sys_catalog.cc:455] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } } }
19:44:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.405481 28970 sys_catalog.cc:458] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [sys.catalog]: This master's current role is: FOLLOWER
19:44:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.406261 28961 sys_catalog.cc:564] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [sys.catalog]: configured and running, proceeding with master startup.
19:44:39.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:39.432945 28982 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:44:39.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:39.434686 28982 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:44:39.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:39.442027 28912 tablet.cc:2369] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:44:39.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.428875 28881 master_runner.cc:186] Error getting master registration for 127.28.12.62:41295: OK, instance_id { permanent_uuid: "f710f73642f14b42812630199af9017d" instance_seqno: 1786045475596732 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:44:39.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.463876 28881 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:44:39.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.811878 28879 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 7
19:44:39.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.832877 28879 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df: Bootstrap starting.
19:44:39.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.839599 28879 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df: Neither blocks nor log segments found. Creating new log.
19:44:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.841457 28879 log.cc:824] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df: Log is configured to *not* fsync() on all Append() calls
19:44:39.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.846065 28879 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df: No bootstrap required, opened a new log
19:44:39.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.871652 28879 raft_consensus.cc:348] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } }
19:44:39.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.873108 28879 raft_consensus.cc:374] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:44:39.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.873711 28879 raft_consensus.cc:729] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cc7f813133994aaab80e1bbd5beea8df, State: Initialized, Role: FOLLOWER
19:44:39.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.874884 28879 consensus_queue.cc:260] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } }
19:44:39.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.877993 28986 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } } }
19:44:39.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.879014 28986 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [sys.catalog]: This master's current role is: FOLLOWER
19:44:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.880060 28970 raft_consensus.cc:482] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:44:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:39.880661 28832 tablet.cc:2369] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:44:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.880633 28970 raft_consensus.cc:504] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } }
19:44:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.882841 28879 sys_catalog.cc:564] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [sys.catalog]: configured and running, proceeding with master startup.
19:44:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.883140 28970 leader_election.cc:288] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f710f73642f14b42812630199af9017d (127.28.12.62:41295), cc7f813133994aaab80e1bbd5beea8df (127.28.12.61:43273)
19:44:39.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.886122 28853 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aafc485e9df84955baa520eec202428d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cc7f813133994aaab80e1bbd5beea8df" is_pre_election: true
19:44:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.887034 28853 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aafc485e9df84955baa520eec202428d in term 0.
19:44:39.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.889719 28909 leader_election.cc:302] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aafc485e9df84955baa520eec202428d, cc7f813133994aaab80e1bbd5beea8df; no voters:
19:44:39.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.891306 28970 raft_consensus.cc:2781] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 0 FOLLOWER]: Leader pre-election won for term 1
19:44:39.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.891844 28970 raft_consensus.cc:482] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:44:39.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.892338 28970 raft_consensus.cc:3037] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 0 FOLLOWER]: Advancing to term 1
19:44:39.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.899005 28970 raft_consensus.cc:504] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } }
19:44:39.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.903263 28853 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aafc485e9df84955baa520eec202428d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cc7f813133994aaab80e1bbd5beea8df"
19:44:39.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.903864 28853 raft_consensus.cc:3037] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [term 0 FOLLOWER]: Advancing to term 1
19:44:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.917060 28970 leader_election.cc:288] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [CANDIDATE]: Term 1 election: Requested vote from peers f710f73642f14b42812630199af9017d (127.28.12.62:41295), cc7f813133994aaab80e1bbd5beea8df (127.28.12.61:43273)
19:44:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.929117 28853 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aafc485e9df84955baa520eec202428d in term 1.
19:44:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.931685 28909 leader_election.cc:302] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aafc485e9df84955baa520eec202428d, cc7f813133994aaab80e1bbd5beea8df; no voters:
19:44:39.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.932905 28970 raft_consensus.cc:2781] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 1 FOLLOWER]: Leader election won for term 1
19:44:39.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.960450 28970 raft_consensus.cc:686] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [term 1 LEADER]: Becoming Leader. State: Replica: aafc485e9df84955baa520eec202428d, State: Running, Role: LEADER
19:44:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.962203 28970 consensus_queue.cc:237] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } }
19:44:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.984588 28990 sys_catalog.cc:455] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [sys.catalog]: SysCatalogTable state changed. Reason: New leader aafc485e9df84955baa520eec202428d. Latest consensus state: current_term: 1 leader_uuid: "aafc485e9df84955baa520eec202428d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } } }
19:44:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:39.985474 28990 sys_catalog.cc:458] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [sys.catalog]: This master's current role is: LEADER
19:44:40.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.003218 28995 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
19:44:40.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.013413 28995 catalog_manager.cc:1484] Initializing Kudu cluster ID...
19:44:40.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:40.032682 29000 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:44:40.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:40.033051 29000 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:44:40.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.076002 28853 raft_consensus.cc:1261] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [term 1 FOLLOWER]: Refusing update from remote peer aafc485e9df84955baa520eec202428d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:44:40.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.078413 28970 consensus_queue.cc:1035] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [LEADER]: Connected to new peer: Peer: permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
19:44:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.020253 28803 master_runner.cc:186] Error getting master registration for 127.28.12.62:41295: OK, instance_id { permanent_uuid: "f710f73642f14b42812630199af9017d" instance_seqno: 1786045475596732 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:44:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.088275 28803 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:44:40.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.115938 29002 mvcc.cc:205] Tried to move back new op lower bound from 7315642286313951232 to 7315642285937348608. Current Snapshot: MvccSnapshot[applied={T|T < 7315642286313951232}]
19:44:40.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.123591 28986 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [sys.catalog]: SysCatalogTable state changed. Reason: New leader aafc485e9df84955baa520eec202428d. Latest consensus state: current_term: 1 leader_uuid: "aafc485e9df84955baa520eec202428d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } } }
19:44:40.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.124459 28986 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [sys.catalog]: This master's current role is: FOLLOWER
19:44:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.137665 28990 sys_catalog.cc:455] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aafc485e9df84955baa520eec202428d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } } }
19:44:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.138559 28990 sys_catalog.cc:458] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [sys.catalog]: This master's current role is: LEADER
19:44:40.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.139987 28986 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aafc485e9df84955baa520eec202428d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } } }
19:44:40.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.141075 28986 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df [sys.catalog]: This master's current role is: FOLLOWER
19:44:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.144851 28995 catalog_manager.cc:1347] Generated new cluster ID: 506660f94e15485d9b8479bd6e651357
19:44:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.145188 28995 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
19:44:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.196166 28995 catalog_manager.cc:1370] Generated new certificate authority record
19:44:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.207916 28995 catalog_manager.cc:1504] Loading token signing keys...
19:44:40.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.244956 28995 catalog_manager.cc:5868] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d: Generated new TSK 0
19:44:40.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.248416 28995 catalog_manager.cc:1514] Initializing in-progress tserver states...
19:44:40.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.262912 28801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } attempt: 7
19:44:40.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.277751 28720 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:40.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:40.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/wal
19:44:40.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/data
19:44:40.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:44:40.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:44:40.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/logs
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/data/info.pb
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.1:0
19:44:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.12.1
19:44:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.1
19:44:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:40.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:44:40.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.337142 28801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d: Bootstrap starting.
19:44:40.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.380539 28801 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d: Neither blocks nor log segments found. Creating new log.
19:44:40.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.395234 28801 log.cc:824] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d: Log is configured to *not* fsync() on all Append() calls
19:44:40.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.433722 28801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d: No bootstrap required, opened a new log
19:44:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.677290 28801 raft_consensus.cc:348] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } }
19:44:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.679688 28801 raft_consensus.cc:374] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:44:40.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.680490 28801 raft_consensus.cc:729] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f710f73642f14b42812630199af9017d, State: Initialized, Role: FOLLOWER
19:44:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.682912 28801 consensus_queue.cc:260] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } }
19:44:40.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.706168 28774 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aafc485e9df84955baa520eec202428d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f710f73642f14b42812630199af9017d"
19:44:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.706358 28775 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aafc485e9df84955baa520eec202428d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f710f73642f14b42812630199af9017d" is_pre_election: true
19:44:40.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.706995 28773 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [term 0 FOLLOWER]: Advancing to term 1
19:44:40.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.718021 28773 raft_consensus.cc:1261] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [term 1 FOLLOWER]: Refusing update from remote peer aafc485e9df84955baa520eec202428d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:44:40.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.705703 29011 sys_catalog.cc:455] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } } }
19:44:40.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.719971 28990 consensus_queue.cc:1035] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [LEADER]: Connected to new peer: Peer: permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:44:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.721518 28801 sys_catalog.cc:564] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [sys.catalog]: configured and running, proceeding with master startup.
19:44:40.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.721871 29011 sys_catalog.cc:458] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [sys.catalog]: This master's current role is: FOLLOWER
19:44:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.766273 29015 mvcc.cc:205] Tried to move back new op lower bound from 7315642286313951232 to 7315642285937348608. Current Snapshot: MvccSnapshot[applied={T|T < 7315642286313951232}]
19:44:40.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.798946 29011 sys_catalog.cc:455] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [sys.catalog]: SysCatalogTable state changed. Reason: New leader aafc485e9df84955baa520eec202428d. Latest consensus state: current_term: 1 leader_uuid: "aafc485e9df84955baa520eec202428d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } } }
19:44:40.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.799666 29011 sys_catalog.cc:458] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [sys.catalog]: This master's current role is: FOLLOWER
19:44:40.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.801231 29012 sys_catalog.cc:455] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aafc485e9df84955baa520eec202428d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } } }
19:44:40.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.801981 29012 sys_catalog.cc:458] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d [sys.catalog]: This master's current role is: FOLLOWER
19:44:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.828991 29031 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
19:44:40.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:40.838742 29009 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:44:40.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:40.839425 29009 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:44:40.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:40.840205 29009 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:44:40.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:40.877436 29009 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:44:40.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:40.878844 29009 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.12.1
19:44:40.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.915311 29009 tablet_server_runner.cc:78] Tablet server non-default flags:
19:44:40.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/data
19:44:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/wal
19:44:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.1:0
19:44:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/data/info.pb
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.1
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.12.1
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/logs
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:44:40.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:44:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:44:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:44:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:44:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:44:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:44:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:44:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.916961 29009 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:44:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:40.919068 29009 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:44:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:40.946360 29034 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:41.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:41.039987 29000 catalog_manager.cc:1259] Loaded cluster ID: 506660f94e15485d9b8479bd6e651357
19:44:41.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:41.040277 29000 catalog_manager.cc:1552] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df: loading cluster ID for follower catalog manager: success
19:44:41.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:41.051788 29000 catalog_manager.cc:1574] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df: acquiring CA information for follower catalog manager: success
19:44:41.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:41.069295 29000 catalog_manager.cc:1602] T 00000000000000000000000000000000 P cc7f813133994aaab80e1bbd5beea8df: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:44:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:41.315665 29037 sys_catalog.cc:455] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aafc485e9df84955baa520eec202428d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f710f73642f14b42812630199af9017d" member_type: VOTER last_known_addr { host: "127.28.12.62" port: 41295 } } peers { permanent_uuid: "cc7f813133994aaab80e1bbd5beea8df" member_type: VOTER last_known_addr { host: "127.28.12.61" port: 43273 } } peers { permanent_uuid: "aafc485e9df84955baa520eec202428d" member_type: VOTER last_known_addr { host: "127.28.12.60" port: 32861 } } }
19:44:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:41.316859 29037 sys_catalog.cc:458] T 00000000000000000000000000000000 P aafc485e9df84955baa520eec202428d [sys.catalog]: This master's current role is: LEADER
19:44:41.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:41.851094 29031 catalog_manager.cc:1259] Loaded cluster ID: 506660f94e15485d9b8479bd6e651357
19:44:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:41.855193 29031 catalog_manager.cc:1552] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d: loading cluster ID for follower catalog manager: success
19:44:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:41.882994 29031 catalog_manager.cc:1574] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d: acquiring CA information for follower catalog manager: success
19:44:41.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:41.894896 29031 catalog_manager.cc:1602] T 00000000000000000000000000000000 P f710f73642f14b42812630199af9017d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:44:42.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:42.878203 28957 debug-util.cc:398] Leaking SignalData structure 0x7b08000c3020 after lost signal to thread 28886
19:44:42.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:42.879142 28957 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbe40 after lost signal to thread 28960
19:44:43.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:42.336189 29032 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 29009
19:44:43.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:43.026834 29009 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 2.083s user 0.695s sys 1.207s
19:44:43.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:43.027180 29009 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 2.084s user 0.695s sys 1.207s
19:44:43.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:40.946456 29033 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:43.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:43.052402 29045 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.058459 29009 server_base.cc:1034] running on GCE node
19:44:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.060300 29009 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:44:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.073725 29009 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:44:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.076056 29009 hybrid_clock.cc:648] HybridClock initialized: now 1786045483076014 us; error 50 us; skew 500 ppm
19:44:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.077185 29009 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:44:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.118155 29009 webserver.cc:460] Webserver started at http://127.28.12.1:37251/ using document root <none> and password file <none>
19:44:43.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.119925 29009 fs_manager.cc:359] Metadata directory not provided
19:44:43.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.120493 29009 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:44:43.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.121269 29009 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:44:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.128605 29009 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/data/instance:
19:44:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34d641cfd5af4068bf5040a806b97fbe"
19:44:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:43 on dist-test-slave-vxj2"
19:44:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.142541 29009 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/wal/instance:
19:44:43.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34d641cfd5af4068bf5040a806b97fbe"
19:44:43.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:43 on dist-test-slave-vxj2"
19:44:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.165917 29009 fs_manager.cc:691] Time spent creating directory manager: real 0.022s user 0.007s sys 0.003s
19:44:43.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.186255 29052 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:44:43.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.187668 29009 fs_manager.cc:725] Time spent opening block manager: real 0.016s user 0.004s sys 0.012s
19:44:43.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.188097 29009 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/wal
19:44:43.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34d641cfd5af4068bf5040a806b97fbe"
19:44:43.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:43 on dist-test-slave-vxj2"
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.188583 29009 fs_report.cc:383] FS layout report
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/wal
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/wal
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/data/data
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:44:43.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:44:43.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:44:43.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:44:43.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:44:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.256978 29009 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:44:43.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.258883 29009 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:44:43.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.259418 29009 kserver.cc:163] Server-wide thread pool size limit: 3276
19:44:43.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.268576 29009 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:44:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.273850 29009 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:44:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.274050 29009 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:44:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.274246 29009 ts_tablet_manager.cc:610] Registered 0 tablets
19:44:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.274474 29009 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:44:43.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.527161 29009 rpc_server.cc:307] RPC server started. Bound to: 127.28.12.1:44823
19:44:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.527266 29167 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.12.1:44823 every 8 connection(s)
19:44:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.531448 29009 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/data/info.pb
19:44:43.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.537083 28720 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 29009
19:44:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.537681 28720 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-0/wal/instance
19:44:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.550137 28720 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/wal
19:44:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/data
19:44:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:44:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:44:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/logs
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/data/info.pb
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.2:0
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.12.2
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.2
19:44:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:43.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:43.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:43.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:43.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:43.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:44:43.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.616171 29168 heartbeater.cc:347] Connected to a master server at 127.28.12.60:32861
19:44:43.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.616899 29168 heartbeater.cc:464] Registering TS with master...
19:44:43.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.618587 29168 heartbeater.cc:511] Master 127.28.12.60:32861 requested a full tablet report, sending...
19:44:43.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.623023 28924 ts_manager.cc:194] Registered new tserver with Master: 34d641cfd5af4068bf5040a806b97fbe (127.28.12.1:44823)
19:44:43.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.628021 28924 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.12.1:46167
19:44:43.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.648123 29169 heartbeater.cc:347] Connected to a master server at 127.28.12.61:43273
19:44:43.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.648538 29169 heartbeater.cc:464] Registering TS with master...
19:44:43.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.649442 29169 heartbeater.cc:511] Master 127.28.12.61:43273 requested a full tablet report, sending...
19:44:43.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.654278 28843 ts_manager.cc:194] Registered new tserver with Master: 34d641cfd5af4068bf5040a806b97fbe (127.28.12.1:44823)
19:44:43.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.665783 29173 heartbeater.cc:347] Connected to a master server at 127.28.12.62:41295
19:44:43.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.666208 29173 heartbeater.cc:464] Registering TS with master...
19:44:43.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.667008 29173 heartbeater.cc:511] Master 127.28.12.62:41295 requested a full tablet report, sending...
19:44:43.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:43.670332 28765 ts_manager.cc:194] Registered new tserver with Master: 34d641cfd5af4068bf5040a806b97fbe (127.28.12.1:44823)
19:44:44.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:44.099663 29172 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:44:44.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:44.100843 29172 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:44:44.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:44.101647 29172 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:44:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:44.159832 29172 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:44:44.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:44.161279 29172 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.12.2
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:44.226445 29172 tablet_server_runner.cc:78] Tablet server non-default flags:
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/data
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/wal
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.2:0
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/data/info.pb
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.2
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.12.2
19:44:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/logs
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:44:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:44.228312 29172 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:44:44.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:44.230826 29172 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:44:44.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:44.261974 29189 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:44.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:44.636575 29168 heartbeater.cc:503] Master 127.28.12.60:32861 was elected leader, sending a full tablet report...
19:44:46.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:45.652164 29188 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 29172
19:44:46.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:46.233969 29188 kernel_stack_watchdog.cc:198] Thread 29172 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 384ms:
19:44:46.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
19:44:46.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
19:44:46.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
19:44:46.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
19:44:46.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:44.263669 29190 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:46.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:46.241240 29172 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.976s user 0.000s sys 0.004s
19:44:46.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:46.241554 29172 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.976s user 0.000s sys 0.004s
19:44:46.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:46.243363 29192 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:46.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.244249 29172 server_base.cc:1034] running on GCE node
19:44:46.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.245913 29172 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:44:46.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.248818 29172 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:44:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.250273 29172 hybrid_clock.cc:648] HybridClock initialized: now 1786045486250185 us; error 90 us; skew 500 ppm
19:44:46.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.251220 29172 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:44:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.259347 29172 webserver.cc:460] Webserver started at http://127.28.12.2:41869/ using document root <none> and password file <none>
19:44:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.260371 29172 fs_manager.cc:359] Metadata directory not provided
19:44:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.260598 29172 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:44:46.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.261045 29172 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:44:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.267120 29172 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/data/instance:
19:44:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a21a72dcac047a5bd0574dad488e972"
19:44:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:46 on dist-test-slave-vxj2"
19:44:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.268357 29172 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/wal/instance:
19:44:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a21a72dcac047a5bd0574dad488e972"
19:44:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:46 on dist-test-slave-vxj2"
19:44:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.277776 29172 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.004s
19:44:46.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.284602 29201 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:44:46.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.285789 29172 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
19:44:46.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.286197 29172 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/wal
19:44:46.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a21a72dcac047a5bd0574dad488e972"
19:44:46.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:46 on dist-test-slave-vxj2"
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.286629 29172 fs_report.cc:383] FS layout report
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/wal
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/wal
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/data/data
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:44:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:44:46.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.333210 29172 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:44:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.334831 29172 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:44:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.335387 29172 kserver.cc:163] Server-wide thread pool size limit: 3276
19:44:46.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.338164 29172 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:44:46.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.342533 29172 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:44:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.342767 29172 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:44:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.343118 29172 ts_tablet_manager.cc:610] Registered 0 tablets
19:44:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.343326 29172 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:44:46.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.555719 29172 rpc_server.cc:307] RPC server started. Bound to: 127.28.12.2:33841
19:44:46.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.555871 29315 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.12.2:33841 every 8 connection(s)
19:44:46.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.558845 29172 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/data/info.pb
19:44:46.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.565352 28720 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 29172
19:44:46.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.566164 28720 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-1/wal/instance
19:44:46.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.588112 28720 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/wal
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/data
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/logs
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/data/info.pb
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:44:46.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:44:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:44:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:44:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.3:0
19:44:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.12.3
19:44:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.3
19:44:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:46.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:46.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:46.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:46.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:46.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:44:46.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.646096 29318 heartbeater.cc:347] Connected to a master server at 127.28.12.62:41295
19:44:46.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.646730 29318 heartbeater.cc:464] Registering TS with master...
19:44:46.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.648296 29318 heartbeater.cc:511] Master 127.28.12.62:41295 requested a full tablet report, sending...
19:44:46.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.646703 29316 heartbeater.cc:347] Connected to a master server at 127.28.12.60:32861
19:44:46.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.650753 29316 heartbeater.cc:464] Registering TS with master...
19:44:46.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.651667 29316 heartbeater.cc:511] Master 127.28.12.60:32861 requested a full tablet report, sending...
19:44:46.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.653131 28765 ts_manager.cc:194] Registered new tserver with Master: 6a21a72dcac047a5bd0574dad488e972 (127.28.12.2:33841)
19:44:46.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.653389 29317 heartbeater.cc:347] Connected to a master server at 127.28.12.61:43273
19:44:46.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.653728 29317 heartbeater.cc:464] Registering TS with master...
19:44:46.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.654594 29317 heartbeater.cc:511] Master 127.28.12.61:43273 requested a full tablet report, sending...
19:44:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.658213 28924 ts_manager.cc:194] Registered new tserver with Master: 6a21a72dcac047a5bd0574dad488e972 (127.28.12.2:33841)
19:44:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.658802 28843 ts_manager.cc:194] Registered new tserver with Master: 6a21a72dcac047a5bd0574dad488e972 (127.28.12.2:33841)
19:44:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:46.661978 28924 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.12.2:57957
19:44:47.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:47.062701 29326 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:44:47.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:47.063385 29326 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:44:47.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:47.063849 29326 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:44:47.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:47.096287 29326 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:44:47.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:47.097188 29326 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.12.3
19:44:47.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:47.135349 29326 tablet_server_runner.cc:78] Tablet server non-default flags:
19:44:47.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:47.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.12.20:46625
19:44:47.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/data
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/wal
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.12.3:0
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/data/info.pb
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.12.3
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.12.62:41295,127.28.12.61:43273,127.28.12.60:32861
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.12.3
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/logs
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:44:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:47.136642 29326 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:44:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:47.138568 29326 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:44:47.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:47.182565 29333 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:47.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:47.666581 29316 heartbeater.cc:503] Master 127.28.12.60:32861 was elected leader, sending a full tablet report...
19:44:48.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:47.184254 29326 server_base.cc:1034] running on GCE node
19:44:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:47.183226 29332 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:48.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:47.186650 29335 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:48.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.442513 29326 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:44:48.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.445142 29326 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:44:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.446683 29326 hybrid_clock.cc:648] HybridClock initialized: now 1786045488446648 us; error 43 us; skew 500 ppm
19:44:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.447651 29326 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:44:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.460060 29326 webserver.cc:460] Webserver started at http://127.28.12.3:33551/ using document root <none> and password file <none>
19:44:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.461346 29326 fs_manager.cc:359] Metadata directory not provided
19:44:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.461622 29326 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:44:48.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.462273 29326 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:44:48.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.468159 29326 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/data/instance:
19:44:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0bc32cf019b410692ca85346ab4a6ee"
19:44:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:48 on dist-test-slave-vxj2"
19:44:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.470257 29326 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/wal/instance:
19:44:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0bc32cf019b410692ca85346ab4a6ee"
19:44:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:48 on dist-test-slave-vxj2"
19:44:48.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.479957 29326 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
19:44:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.487166 29343 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:44:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.488461 29326 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
19:44:48.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.488859 29326 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/wal
19:44:48.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0bc32cf019b410692ca85346ab4a6ee"
19:44:48.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:48 on dist-test-slave-vxj2"
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.489388 29326 fs_report.cc:383] FS layout report
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/wal
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/wal
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/data/data
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:44:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:44:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.530113 29326 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:44:48.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.531669 29326 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:44:48.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.532236 29326 kserver.cc:163] Server-wide thread pool size limit: 3276
19:44:48.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.534986 29326 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:44:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.539561 29326 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:44:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.539762 29326 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:44:48.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.539958 29326 ts_tablet_manager.cc:610] Registered 0 tablets
19:44:48.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:48.540100 29326 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:44:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.139387 29326 rpc_server.cc:307] RPC server started. Bound to: 127.28.12.3:36651
19:44:49.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.140134 29457 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.12.3:36651 every 8 connection(s)
19:44:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.142907 29326 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/data/info.pb
19:44:49.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.146742 28720 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 29326
19:44:49.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.147351 28720 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster1133231897502307326/ts-2/wal/instance
19:44:49.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.240964 29458 heartbeater.cc:347] Connected to a master server at 127.28.12.60:32861
19:44:49.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.241871 29458 heartbeater.cc:464] Registering TS with master...
19:44:49.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.243829 29458 heartbeater.cc:511] Master 127.28.12.60:32861 requested a full tablet report, sending...
19:44:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.246269 29459 heartbeater.cc:347] Connected to a master server at 127.28.12.61:43273
19:44:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.246868 29459 heartbeater.cc:464] Registering TS with master...
19:44:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.248168 29459 heartbeater.cc:511] Master 127.28.12.61:43273 requested a full tablet report, sending...
19:44:49.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.256780 28843 ts_manager.cc:194] Registered new tserver with Master: c0bc32cf019b410692ca85346ab4a6ee (127.28.12.3:36651)
19:44:49.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.261960 29462 heartbeater.cc:347] Connected to a master server at 127.28.12.62:41295
19:44:49.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.262554 29462 heartbeater.cc:464] Registering TS with master...
19:44:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.263872 29462 heartbeater.cc:511] Master 127.28.12.62:41295 requested a full tablet report, sending...
19:44:49.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.266654 28924 ts_manager.cc:194] Registered new tserver with Master: c0bc32cf019b410692ca85346ab4a6ee (127.28.12.3:36651)
19:44:49.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.267917 28765 ts_manager.cc:194] Registered new tserver with Master: c0bc32cf019b410692ca85346ab4a6ee (127.28.12.3:36651)
19:44:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.271188 28924 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.28.12.3:35607
19:44:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:49.294516 28720 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
19:44:49.317 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:44:49.318 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
19:44:49.330 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.12.62"
port: 41295
}
bound_http_address {
host: "127.28.12.62"
port: 37097
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.12.61"
port: 43273
}
bound_http_address {
host: "127.28.12.61"
port: 45433
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.12.60"
port: 32861
}
bound_http_address {
host: "127.28.12.60"
port: 36435
}
}
}
19:44:49.342 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:44:49.345 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.28.12.1"
port: 44823
}
bound_http_address {
host: "127.28.12.1"
port: 37251
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.28.12.2"
port: 33841
}
bound_http_address {
host: "127.28.12.2"
port: 41869
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.28.12.3"
port: 36651
}
bound_http_address {
host: "127.28.12.3"
port: 33551
}
}
}
19:44:49.345 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
19:44:49.650 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
19:44:49.811 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskBYv0IG/test-tmp
19:44:49.813 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:44:49.813 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskBYv0IG/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:44:49.818 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
19:44:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:50.278386 29458 heartbeater.cc:503] Master 127.28.12.60:32861 was elected leader, sending a full tablet report...
19:44:50.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:44:50.508 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:44:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:50.512238 29469 mini_oidc.cc:152] Starting JWKS server
19:44:50.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:50.522189 29469 webserver.cc:460] Webserver started at http://127.0.0.1:35067/ using document root <none> and password file <none>
19:44:50.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:50.522899 29469 mini_oidc.cc:189] Starting OIDC Discovery server
19:44:50.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:50.530135 29469 webserver.cc:460] Webserver started at http://127.0.0.1:35707/ using document root <none> and password file <none>
19:44:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:50.582706 29312 debug-util.cc:398] Leaking SignalData structure 0x7b08000af120 after lost signal to thread 29180
19:44:50.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:50.585075 29312 debug-util.cc:398] Leaking SignalData structure 0x7b08000afd60 after lost signal to thread 29315
19:44:50.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
19:44:50.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/principal' for realm 'KRBTEST.COM',
19:44:50.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
19:44:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:44:50 dist-test-slave-vxj2 krb5kdc[29481](info): setting up network...
19:44:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
19:44:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:44:50 dist-test-slave-vxj2 krb5kdc[29481](info): set up 2 sockets
19:44:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:44:50 dist-test-slave-vxj2 krb5kdc[29481](info): commencing operation
19:44:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
19:44:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:51.837092 29469 mini_kdc.cc:121] Time spent starting KDC: real 1.258s user 0.001s sys 0.041s
19:44:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
19:44:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:44:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
19:44:51.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
19:44:51.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:44:51.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
19:44:51.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
19:44:51.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:44:51.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
19:44:51.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:44:51.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/test-user.keytab.
19:44:51.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/test-user.keytab.
19:44:52.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:44:52 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045492, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:44:52.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:44:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T19:44:52Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:44:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T19:44:52Z Disabled control of system clock
19:44:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.199.126@KRBTEST.COM; defaulting to no policy
19:44:52.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:44:52.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.199.126@KRBTEST.COM" created.
19:44:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:44:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.199.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.126.keytab.
19:44:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.199.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.126.keytab.
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:52.128381 29469 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/logs
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data/info.pb
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:44:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:44:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.126:34497
19:44:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.126
19:44:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.126.keytab
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.126
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:44:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.818720 29496 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:44:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.819607 29496 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:44:52.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.820407 29496 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:44:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.856580 29496 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:44:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.856941 29496 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:44:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.857390 29496 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:44:52.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.857729 29496 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:44:52.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.858049 29496 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:44:52.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.858372 29496 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:44:52.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.858609 29496 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35067/jwks.json
19:44:52.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:52.900350 29496 master_runner.cc:386] Master server non-default flags:
19:44:52.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.126.keytab
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.126
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.126:34497
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data/info.pb
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.126
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/logs
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:44:52.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:44:52.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:44:52.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:44:52.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:52.902040 29496 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:44:52.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:52.903546 29496 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:44:52.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.943675 29502 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:54.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.946049 29503 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:54.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:52.958132 29505 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:54.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:54.285665 29504 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:44:54.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.285759 29496 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:44:54.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.289618 29496 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:44:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.292236 29496 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:44:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.293684 29496 hybrid_clock.cc:648] HybridClock initialized: now 1786045494293577 us; error 91 us; skew 500 ppm
19:44:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.294596 29496 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:44:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:44:54 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045494, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:44:54.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.312340 29496 init.cc:377] Logged in from keytab as kudu/127.28.199.126@KRBTEST.COM (short username kudu)
19:44:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.318011 29496 webserver.cc:460] Webserver started at http://127.28.199.126:35187/ using document root <none> and password file <none>
19:44:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.319098 29496 fs_manager.cc:359] Metadata directory not provided
19:44:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.319344 29496 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:44:54.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.319896 29496 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:44:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.325556 29496 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data/instance:
19:44:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d102020153984228a0194a28ea6e894b"
19:44:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:54 on dist-test-slave-vxj2"
19:44:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.326932 29496 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal/instance:
19:44:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d102020153984228a0194a28ea6e894b"
19:44:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:54 on dist-test-slave-vxj2"
19:44:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.334730 29496 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
19:44:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.340901 29517 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:44:54.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.342238 29496 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.342670 29496 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d102020153984228a0194a28ea6e894b"
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:54 on dist-test-slave-vxj2"
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.343072 29496 fs_report.cc:383] FS layout report
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data/data
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:44:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:44:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:44:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:44:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:44:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:44:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:44:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:44:54.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.383960 29496 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:44:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.390535 29496 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:44:54.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.390995 29496 kserver.cc:163] Server-wide thread pool size limit: 3276
19:44:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.603731 29496 rpc_server.cc:307] RPC server started. Bound to: 127.28.199.126:34497
19:44:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.604058 29569 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.199.126:34497 every 8 connection(s)
19:44:54.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.608687 29496 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data/info.pb
19:44:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.612960 29469 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 29496
19:44:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.613864 29469 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal/instance
19:44:54.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.625501 29570 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:44:54.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.660802 29570 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } has no permanent_uuid. Determining permanent_uuid...
19:44:54.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.199.125@KRBTEST.COM; defaulting to no policy
19:44:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:44:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.199.125@KRBTEST.COM" created.
19:44:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:44:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.199.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.125.keytab.
19:44:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.199.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.125.keytab.
19:44:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:54.728515 29469 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.28.199.125: real 0.106s user 0.002s sys 0.021s
19:44:54.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.731456 29469 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:54.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:54.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal
19:44:54.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data
19:44:54.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:44:54.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:44:54.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:44:54.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/logs
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data/info.pb
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.125:46813
19:44:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.125
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.125.keytab
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.125
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:44:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:44:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:44:54 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045494, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.126@KRBTEST.COM for kudu/127.28.199.126@KRBTEST.COM
19:44:54.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.785529 29570 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } has no permanent_uuid. Determining permanent_uuid...
19:44:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:54.790412 29520 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:44:54.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:54.803049 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.125:46813: Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:44:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.852739 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } attempt: 1
19:44:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:54.859923 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.125:46813: Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:44:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:54.928530 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } attempt: 2
19:44:54.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:54.936030 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.125:46813: Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:44:55.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:55.027704 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } attempt: 3
19:44:55.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.035333 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.125:46813: Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:44:55.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:55.178886 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } attempt: 4
19:44:55.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.186550 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.125:46813: Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:44:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.370277 29576 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:44:55.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.371083 29576 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:44:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.371846 29576 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:44:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.410626 29576 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:44:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.410953 29576 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:44:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.411384 29576 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:44:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.411731 29576 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:44:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.412068 29576 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:44:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.412391 29576 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:44:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.412627 29576 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35067/jwks.json
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:55.456878 29576 master_runner.cc:386] Master server non-default flags:
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.125.keytab
19:44:55.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.125
19:44:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.125:46813
19:44:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:44:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:44:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:44:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:44:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:44:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data/info.pb
19:44:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.125
19:44:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:44:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:44:55.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:44:55.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:55.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/logs
19:44:55.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:55.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:44:55.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:44:55.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:44:55.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:44:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:44:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:44:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:44:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:44:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:44:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:55.458864 29576 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:44:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:55.460429 29576 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:44:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:55.486635 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } attempt: 5
19:44:55.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.496114 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.125:46813: Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:44:55.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.555106 29584 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:56.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:56.043823 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } attempt: 6
19:44:56.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:56.058451 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.125:46813: Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:44:56.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:56.561445 28797 debug-util.cc:398] Leaking SignalData structure 0x7b0800097220 after lost signal to thread 28733
19:44:56.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:56.562515 28797 debug-util.cc:398] Leaking SignalData structure 0x7b0800082de0 after lost signal to thread 28800
19:44:57.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.119117 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } attempt: 7
19:44:57.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:57.148690 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.125:46813: Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:44:57.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.578135 29587 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:57.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:55.567819 29585 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:57.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:57.320267 29586 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1763 milliseconds
19:44:57.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.320387 29576 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:44:57.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.321861 29576 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:44:57.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.325395 29576 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:44:57.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.328877 29576 hybrid_clock.cc:648] HybridClock initialized: now 1786045497328824 us; error 71 us; skew 500 ppm
19:44:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.330323 29576 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:44:57.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:44:57 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045497, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:44:57.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.344139 29576 init.cc:377] Logged in from keytab as kudu/127.28.199.125@KRBTEST.COM (short username kudu)
19:44:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.357669 29576 webserver.cc:460] Webserver started at http://127.28.199.125:38103/ using document root <none> and password file <none>
19:44:57.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.358680 29576 fs_manager.cc:359] Metadata directory not provided
19:44:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.358917 29576 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:44:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.359396 29576 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:44:57.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.365346 29576 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data/instance:
19:44:57.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e653af98c80247fbae18a0fe8ce38645"
19:44:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:57 on dist-test-slave-vxj2"
19:44:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.366449 29576 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal/instance:
19:44:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e653af98c80247fbae18a0fe8ce38645"
19:44:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:57 on dist-test-slave-vxj2"
19:44:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.379747 29576 fs_manager.cc:691] Time spent creating directory manager: real 0.013s user 0.004s sys 0.008s
19:44:57.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.394587 29605 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:44:57.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.396314 29576 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.002s sys 0.006s
19:44:57.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.396739 29576 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal
19:44:57.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e653af98c80247fbae18a0fe8ce38645"
19:44:57.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:57 on dist-test-slave-vxj2"
19:44:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.397151 29576 fs_report.cc:383] FS layout report
19:44:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:44:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal
19:44:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal
19:44:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data/data
19:44:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:44:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:44:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:44:57.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:44:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:44:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:44:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:44:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:44:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:44:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:44:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:44:57.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.464716 29576 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:44:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.472975 29576 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:44:57.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.473789 29576 kserver.cc:163] Server-wide thread pool size limit: 3276
19:44:57.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.802091 29576 rpc_server.cc:307] RPC server started. Bound to: 127.28.199.125:46813
19:44:57.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.802510 29657 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.199.125:46813 every 8 connection(s)
19:44:57.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.805708 29576 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data/info.pb
19:44:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.807018 29469 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 29576
19:44:57.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.807652 29469 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal/instance
19:44:57.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.824383 29658 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:44:57.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.854651 29658 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } has no permanent_uuid. Determining permanent_uuid...
19:44:57.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.199.124@KRBTEST.COM; defaulting to no policy
19:44:57.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:44:57.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.199.124@KRBTEST.COM" created.
19:44:57.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:44:57 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045497, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.125@KRBTEST.COM for kudu/127.28.199.126@KRBTEST.COM
19:44:57.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:44:57.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.199.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.124.keytab.
19:44:57.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.199.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.124.keytab.
19:44:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.907725 29469 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:44:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal
19:44:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data
19:44:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:44:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:44:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:44:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/logs
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data/info.pb
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.124:34413
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.124
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.124.keytab
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.124
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:44:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:44:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:44:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:44:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:44:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:44:57.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.917363 29658 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } has no permanent_uuid. Determining permanent_uuid...
19:44:57.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:44:57 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045497, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.125@KRBTEST.COM for kudu/127.28.199.125@KRBTEST.COM
19:44:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:57.960556 29658 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } has no permanent_uuid. Determining permanent_uuid...
19:44:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:57.964983 29608 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:57.977974 29658 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:58.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:58.027474 29658 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 1
19:44:58.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.034035 29658 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:58.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:58.102531 29658 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 2
19:44:58.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.109081 29658 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:58.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:58.200692 29658 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 3
19:44:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.209200 29658 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:58.352754 29658 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 4
19:44:58.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.378275 29658 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:58.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.600867 29665 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:44:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.602068 29665 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:44:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.602854 29665 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:44:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.639811 29665 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:44:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.640126 29665 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:44:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.640528 29665 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:44:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.640874 29665 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:44:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.641225 29665 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:44:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.641578 29665 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:44:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.641825 29665 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35067/jwks.json
19:44:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:58.677868 29658 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 5
19:44:58.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.685226 29658 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:58.688366 29665 master_runner.cc:386] Master server non-default flags:
19:44:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:44:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.124.keytab
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.124
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.124:34413
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data/info.pb
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.124
19:44:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/logs
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:44:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:58.690016 29665 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:44:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:58.691378 29665 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:44:58.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.744622 29675 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:44:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:59.232857 29658 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 6
19:44:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:59.239198 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } attempt: 8
19:44:59.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:59.286242 29658 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:59.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:44:59 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045494, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.126@KRBTEST.COM for kudu/127.28.199.125@KRBTEST.COM
19:44:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:59.332340 29570 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } has no permanent_uuid. Determining permanent_uuid...
19:44:59.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:59.364491 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:59.430368 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 1
19:44:59.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:59.437394 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:59.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:59.490980 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 2
19:44:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:59.502204 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:59.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:59.578963 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 3
19:44:59.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:59.586771 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:44:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:44:59.742475 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 4
19:44:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:59.757146 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:45:00.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.053960 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 5
19:45:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:00.058365 29520 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111) [suppressed 12 similar messages]
19:45:00.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:00.063644 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:45:00.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.349277 29658 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 7
19:45:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:00.399232 29658 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:45:00.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:44:58.752070 29674 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:00.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:00.485725 29676 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1739 milliseconds
19:45:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.486905 29665 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:45:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:00.487111 29678 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:00.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.490113 29665 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:45:00.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.494903 29665 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:45:00.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.496603 29665 hybrid_clock.cc:648] HybridClock initialized: now 1786045500496564 us; error 57 us; skew 500 ppm
19:45:00.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.497592 29665 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:45:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:00 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045500, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:45:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.509800 29665 init.cc:377] Logged in from keytab as kudu/127.28.199.124@KRBTEST.COM (short username kudu)
19:45:00.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.515772 29665 webserver.cc:460] Webserver started at http://127.28.199.124:44775/ using document root <none> and password file <none>
19:45:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.516853 29665 fs_manager.cc:359] Metadata directory not provided
19:45:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.517124 29665 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:45:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.517657 29665 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:45:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.522565 29665 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data/instance:
19:45:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db4f06f81a0f40f6a51354aa78aac6b8"
19:45:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:45:00 on dist-test-slave-vxj2"
19:45:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.523635 29665 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal/instance:
19:45:00.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db4f06f81a0f40f6a51354aa78aac6b8"
19:45:00.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:45:00 on dist-test-slave-vxj2"
19:45:00.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.532428 29665 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.003s sys 0.007s
19:45:00.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.539536 29694 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:45:00.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.541109 29665 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
19:45:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.541834 29665 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal
19:45:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db4f06f81a0f40f6a51354aa78aac6b8"
19:45:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:45:00 on dist-test-slave-vxj2"
19:45:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.542251 29665 fs_report.cc:383] FS layout report
19:45:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:45:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal
19:45:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data/data
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:45:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:45:00.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.581753 29665 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:45:00.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.585350 29570 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } attempt: 6
19:45:00.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.589663 29665 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:45:00.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.590237 29665 kserver.cc:163] Server-wide thread pool size limit: 3276
19:45:00.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:00.593713 29570 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.199.124:34413: Network error: Client connection negotiation failed: client connection to 127.28.199.124:34413: connect: Connection refused (error 111)
19:45:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.880136 29665 rpc_server.cc:307] RPC server started. Bound to: 127.28.199.124:34413
19:45:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.880154 29750 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.199.124:34413 every 8 connection(s)
19:45:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.883381 29665 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data/info.pb
19:45:00.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.892736 29469 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 29665
19:45:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.895890 29469 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal/instance
19:45:00.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.897025 29751 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:45:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:00.935662 29751 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } has no permanent_uuid. Determining permanent_uuid...
19:45:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:00 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045492, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.199.124@KRBTEST.COM
19:45:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:00 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045500, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.124@KRBTEST.COM for kudu/127.28.199.126@KRBTEST.COM
19:45:01.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.003948 29751 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } has no permanent_uuid. Determining permanent_uuid...
19:45:01.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:01 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045500, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.124@KRBTEST.COM for kudu/127.28.199.125@KRBTEST.COM
19:45:01.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:01 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045492, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.199.125@KRBTEST.COM
19:45:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.037717 29751 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } has no permanent_uuid. Determining permanent_uuid...
19:45:01.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:01 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045500, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.124@KRBTEST.COM for kudu/127.28.199.124@KRBTEST.COM
19:45:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:01 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045492, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.199.126@KRBTEST.COM
19:45:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.073545 29751 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: Bootstrap starting.
19:45:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.084955 29469 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
19:45:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.086205 29751 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: Neither blocks nor log segments found. Creating new log.
19:45:01.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.090902 29751 log.cc:824] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: Log is configured to *not* fsync() on all Append() calls
19:45:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.098975 29751 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: No bootstrap required, opened a new log
19:45:01.108 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:45:01.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
19:45:01.128 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
19:45:01.130 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
19:45:01.135 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.199.126"
port: 34497
}
bound_http_address {
host: "127.28.199.126"
port: 35187
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.199.125"
port: 46813
}
bound_http_address {
host: "127.28.199.125"
port: 38103
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.199.124"
port: 34413
}
bound_http_address {
host: "127.28.199.124"
port: 44775
}
}
}
19:45:01.135 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:45:01.136 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:45:01.136 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:45:01.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:45:01.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.151553 29751 raft_consensus.cc:348] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:01.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.153568 29751 raft_consensus.cc:374] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:45:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.154187 29751 raft_consensus.cc:729] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db4f06f81a0f40f6a51354aa78aac6b8, State: Initialized, Role: FOLLOWER
19:45:01.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.156032 29751 consensus_queue.cc:260] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.164103 29760 sys_catalog.cc:455] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } } }
19:45:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.165804 29760 sys_catalog.cc:458] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [sys.catalog]: This master's current role is: FOLLOWER
19:45:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.167860 29751 sys_catalog.cc:564] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [sys.catalog]: configured and running, proceeding with master startup.
19:45:01.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:01 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045501, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:45:01.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:45:01.213 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:45:01.215 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5cc
19:45:01.227 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:45:01.228 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5cc
19:45:01.230 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:45:01.242 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
19:45:01.244 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:45:01.245 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:45:01.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.246619 29773 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.247447 29773 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:01.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.225889 29665 master_runner.cc:186] Error getting master registration for 127.28.199.126:34497: OK, instance_id { permanent_uuid: "d102020153984228a0194a28ea6e894b" instance_seqno: 1786045494521736 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:45:01.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.280720 29665 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:45:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:01 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045501, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:45:01.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:45:01.335 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:45:01.335 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5cc
19:45:01.336 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:45:01.337 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.199.125:46813
19:45:01.338 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
19:45:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.339228 29469 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu with pid 29576
19:45:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.347905 29703 tablet.cc:2369] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:45:01.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:45:01.370 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.199.124:34413
19:45:01.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
19:45:01.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.377894 29469 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu with pid 29665
19:45:01.408 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:45:01.409 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.199.126:34497
19:45:01.409 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
19:45:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.409987 29469 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu with pid 29496
19:45:01.437 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:45:01.438 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.199.125:46813
19:45:01.440 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.441994 29469 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/logs
19:45:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data/info.pb
19:45:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:45:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:45:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:45:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:45:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:45:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:45:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:45:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:45:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:45:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:45:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:45:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:45:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.125:46813
19:45:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.125
19:45:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38103
19:45:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.125.keytab
19:45:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.125
19:45:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:45:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:45:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:45:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:45:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:45:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:45:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:45:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:45:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:45:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:45:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:45:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:45:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.786950 29778 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:45:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.787534 29778 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:45:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.788156 29778 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:45:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.824849 29778 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:45:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.825111 29778 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:45:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.825578 29778 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:45:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.825807 29778 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:45:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.826009 29778 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:45:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.826200 29778 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:45:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.826346 29778 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35067/jwks.json
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.865772 29778 master_runner.cc:386] Master server non-default flags:
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.125.keytab
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.125
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.125:46813
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:45:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data/info.pb
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.125
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38103
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/logs
19:45:01.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.866988 29778 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:45:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:01.867997 29778 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:45:01.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.892509 29787 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:03.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.892935 29785 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:03.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:01.894157 29784 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.137887 29786 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:45:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.137939 29778 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:45:03.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.142185 29778 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:45:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.145697 29778 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:45:03.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.147147 29778 hybrid_clock.cc:648] HybridClock initialized: now 1786045503147033 us; error 129 us; skew 500 ppm
19:45:03.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.148030 29778 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:45:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:03 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045503, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:45:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.154452 29778 init.cc:377] Logged in from keytab as kudu/127.28.199.125@KRBTEST.COM (short username kudu)
19:45:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.159960 29778 webserver.cc:460] Webserver started at http://127.28.199.125:38103/ using document root <none> and password file <none>
19:45:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.161095 29778 fs_manager.cc:359] Metadata directory not provided
19:45:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.161496 29778 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:45:03.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.164633 28797 debug-util.cc:398] Leaking SignalData structure 0x7b080009c820 after lost signal to thread 28733
19:45:03.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.165325 28797 debug-util.cc:398] Leaking SignalData structure 0x7b08000a7920 after lost signal to thread 28800
19:45:03.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.171700 29778 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.005s sys 0.004s
19:45:03.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.177348 29795 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:45:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.178779 29778 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
19:45:03.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.179183 29778 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal
19:45:03.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e653af98c80247fbae18a0fe8ce38645"
19:45:03.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:57 on dist-test-slave-vxj2"
19:45:03.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.183004 29778 fs_report.cc:383] FS layout report
19:45:03.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:45:03.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal
19:45:03.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal
19:45:03.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data/data
19:45:03.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:45:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:45:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:45:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:45:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:45:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:45:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:45:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:45:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:45:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:45:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:45:03.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.235198 29778 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:45:03.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.242163 29778 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:45:03.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.242688 29778 kserver.cc:163] Server-wide thread pool size limit: 3276
19:45:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.344915 29778 rpc_server.cc:307] RPC server started. Bound to: 127.28.199.125:46813
19:45:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.345386 29848 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.199.125:46813 every 8 connection(s)
19:45:03.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.348268 29778 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/data/info.pb
19:45:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.351035 29469 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 29778
19:45:03.352 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:45:03.352 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.199.124:34413
19:45:03.352 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
19:45:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.355304 29469 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:45:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/logs
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data/info.pb
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.124:34413
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.124
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44775
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.124.keytab
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.124
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:45:03.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:45:03.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:45:03.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:45:03.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:45:03.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:45:03.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:45:03.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.384889 29849 sys_catalog.cc:263] Verifying existing consensus state
19:45:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 19:45:03.386802 29849 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:45:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.387712 29778 master.cc:537] Master@127.28.199.125:46813 shutting down...
19:45:03.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.458846 29778 master.cc:552] Master@127.28.199.125:46813 shutdown complete.
19:45:03.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:45:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.777405 29851 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:45:03.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.777984 29851 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:45:03.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.778584 29851 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:45:03.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.815996 29851 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:45:03.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.816308 29851 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:45:03.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.816623 29851 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:45:03.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.816984 29851 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:45:03.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.817404 29851 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:45:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.817775 29851 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:45:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.818011 29851 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35067/jwks.json
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.856927 29851 master_runner.cc:386] Master server non-default flags:
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.124.keytab
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.124
19:45:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.124:34413
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data/info.pb
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.124
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44775
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/logs
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:45:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:45:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:45:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:45:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.858176 29851 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:45:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.859249 29851 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:45:03.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.878381 29858 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:03.880484 29851 server_base.cc:1034] running on GCE node
19:45:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.879472 29859 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:03.878433 29861 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:05.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.194183 29851 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:45:05.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.197853 29851 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:45:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.199527 29851 hybrid_clock.cc:648] HybridClock initialized: now 1786045505199477 us; error 59 us; skew 500 ppm
19:45:05.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.200660 29851 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:45:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:05 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045505, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:45:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.209213 29851 init.cc:377] Logged in from keytab as kudu/127.28.199.124@KRBTEST.COM (short username kudu)
19:45:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.215896 29851 webserver.cc:460] Webserver started at http://127.28.199.124:44775/ using document root <none> and password file <none>
19:45:05.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.217432 29851 fs_manager.cc:359] Metadata directory not provided
19:45:05.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.217777 29851 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:45:05.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.229769 29851 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
19:45:05.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.234844 29871 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:45:05.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.236066 29851 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.236492 29851 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db4f06f81a0f40f6a51354aa78aac6b8"
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:45:00 on dist-test-slave-vxj2"
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.239039 29851 fs_report.cc:383] FS layout report
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/wal
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data/data
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:45:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:45:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:45:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:45:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:45:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:45:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:45:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:45:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:45:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:45:05.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.276448 29851 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:45:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.283041 29851 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:45:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.283612 29851 kserver.cc:163] Server-wide thread pool size limit: 3276
19:45:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.361827 29851 rpc_server.cc:307] RPC server started. Bound to: 127.28.199.124:34413
19:45:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.361963 29923 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.199.124:34413 every 8 connection(s)
19:45:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.364974 29851 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-2/data/info.pb
19:45:05.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.374116 29469 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 29851
19:45:05.375 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:45:05.375 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.199.126:34497
19:45:05.376 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.377990 29469 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/logs
19:45:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data/info.pb
19:45:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:45:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:45:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:45:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:45:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:45:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:45:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:45:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.126:34497
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.126
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35187
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.126.keytab
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.126
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:45:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:45:05.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:45:05.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:45:05.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:45:05.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.378721 29924 sys_catalog.cc:263] Verifying existing consensus state
19:45:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.390297 29924 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: Bootstrap starting.
19:45:05.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.434003 29924 log.cc:824] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: Log is configured to *not* fsync() on all Append() calls
19:45:05.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.438966 29924 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:45:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.439953 29924 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: Bootstrap complete.
19:45:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.466763 29924 raft_consensus.cc:348] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.469540 29924 raft_consensus.cc:374] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:45:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.469867 29924 raft_consensus.cc:729] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db4f06f81a0f40f6a51354aa78aac6b8, State: Initialized, Role: FOLLOWER
19:45:05.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.470988 29924 consensus_queue.cc:260] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.474124 29928 sys_catalog.cc:455] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } } }
19:45:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.475636 29928 sys_catalog.cc:458] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [sys.catalog]: This master's current role is: FOLLOWER
19:45:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.476099 29924 sys_catalog.cc:564] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [sys.catalog]: configured and running, proceeding with master startup.
19:45:05.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.487501 29928 raft_consensus.cc:482] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:45:05.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.488251 29928 raft_consensus.cc:504] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:05.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.496644 29928 leader_election.cc:288] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d102020153984228a0194a28ea6e894b (127.28.199.126:34497), e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813)
19:45:05.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.504385 29874 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111)
19:45:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.511297 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d102020153984228a0194a28ea6e894b (127.28.199.126:34497): Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111)
19:45:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.512578 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813): Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:45:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.513358 29874 leader_election.cc:302] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db4f06f81a0f40f6a51354aa78aac6b8; no voters: d102020153984228a0194a28ea6e894b, e653af98c80247fbae18a0fe8ce38645
19:45:05.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.514775 29928 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:45:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.539109 29877 tablet.cc:2369] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:45:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.548689 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.549070 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.525899 29851 master_runner.cc:186] Error getting master registration for 127.28.199.126:34497: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111),
19:45:05.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.557725 29851 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:45:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.798394 29926 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:45:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.799000 29926 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:45:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.799660 29926 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:45:05.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.855743 29926 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:45:05.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.856109 29926 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:45:05.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.856577 29926 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:45:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.856930 29926 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:45:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.857285 29926 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:45:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.857636 29926 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:45:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.857904 29926 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35067/jwks.json
19:45:05.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.910341 29926 master_runner.cc:386] Master server non-default flags:
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.199.84:33359
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.199.126:34497,127.28.199.125:46813,127.28.199.124:34413
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/krb5kdc/kudu_127.28.199.126.keytab
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.199.126
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:45:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.199.126:34497
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35067/jwks.json
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data/info.pb
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.199.126
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35187
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:45:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/logs
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 18:52:39 UTC on ccycloud.quasar-wuhknf.root.comops.site
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4571035
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.912122 29926 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:45:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.913650 29926 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:45:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.936403 29950 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:06.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:06.555559 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:06.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:06.556097 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:07.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.144508 29958 raft_consensus.cc:482] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:45:07.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.144973 29958 raft_consensus.cc:504] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:07.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.147145 29958 leader_election.cc:288] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d102020153984228a0194a28ea6e894b (127.28.199.126:34497), e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813)
19:45:07.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:07.156522 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d102020153984228a0194a28ea6e894b (127.28.199.126:34497): Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111)
19:45:07.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:07.157444 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813): Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:45:07.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.157768 29874 leader_election.cc:302] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db4f06f81a0f40f6a51354aa78aac6b8; no voters: d102020153984228a0194a28ea6e894b, e653af98c80247fbae18a0fe8ce38645
19:45:07.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.158524 29958 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:45:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.938275 29951 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:05.941071 29926 server_base.cc:1034] running on GCE node
19:45:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:05.938119 29953 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:45:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.280650 29926 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:45:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.284178 29926 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:45:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.285614 29926 hybrid_clock.cc:648] HybridClock initialized: now 1786045507285727 us; error 201 us; skew 500 ppm
19:45:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.286608 29926 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:45:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:07 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045507, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:45:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.297847 29926 init.cc:377] Logged in from keytab as kudu/127.28.199.126@KRBTEST.COM (short username kudu)
19:45:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.303920 29926 webserver.cc:460] Webserver started at http://127.28.199.126:35187/ using document root <none> and password file <none>
19:45:07.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.305130 29926 fs_manager.cc:359] Metadata directory not provided
19:45:07.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.305462 29926 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:45:07.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.316622 29926 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.005s sys 0.000s
19:45:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.322387 29967 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:45:07.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.323859 29926 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.324194 29926 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data,/tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d102020153984228a0194a28ea6e894b"
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 19:44:54 on dist-test-slave-vxj2"
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.327076 29926 fs_report.cc:383] FS layout report
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data/data
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:45:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:45:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:45:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:45:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:45:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:45:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:45:07.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.397933 29926 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:45:07.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.405728 29926 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:45:07.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.406505 29926 kserver.cc:163] Server-wide thread pool size limit: 3276
19:45:07.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.514305 29926 rpc_server.cc:307] RPC server started. Bound to: 127.28.199.126:34497
19:45:07.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.514487 30019 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.199.126:34497 every 8 connection(s)
19:45:07.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.517640 29926 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/data/info.pb
19:45:07.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.524300 29469 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu as pid 29926
19:45:07.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:45:07.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.532461 30020 sys_catalog.cc:263] Verifying existing consensus state
19:45:07.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 19:45:07.533473 30020 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:45:07.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.533982 29926 master.cc:537] Master@127.28.199.126:34497 shutting down...
19:45:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:07.560858 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:07.562271 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:07.602 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:07.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:07.618336 29926 master.cc:552] Master@127.28.199.126:34497 shutdown complete.
19:45:07.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskBYv0IG/test-tmp/mini-kudu-cluster5680930097596983181/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:45:07.690 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.199.124:34413(127.28.199.124:34413)] connecting to peer
19:45:07.691 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:07.749 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.199.124:34413(127.28.199.124:34413)] Successfully connected to peer
19:45:07.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:07.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:07.755 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:07.756 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:07.757 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:07.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:07.831 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
19:45:07.839 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.199.124
19:45:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:08 dist-test-slave-vxj2 krb5kdc[29481](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045501, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.199.124@KRBTEST.COM
19:45:08.173 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x97d5edef, L:/127.0.0.1:48134 - R:/127.28.199.124:34413] using SASL/GSSAPI
19:45:08.195 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:08.212 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:08.217 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:08.222 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.223 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.223 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.224 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.224 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.227 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:08.251 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:08.253 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:08.256 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.256 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.257 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.257 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.257 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.264 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:08.270 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:08.272 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:08.276 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.276 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.277 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.280 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:08.311 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:08.313 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:08.315 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.315 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.316 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.316 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.316 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.317 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:08.331 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:08.333 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:08.335 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.335 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.335 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.335 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.336 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.336 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.338 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:08.371 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:08.372 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:08.375 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.376 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.377 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.377 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.380 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:08.451 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:08.453 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:08.455 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.455 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.456 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.456 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.456 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.457 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.458 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:08.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:08.565775 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:08.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:08.566056 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:08.691 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:08.693 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:08.695 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.695 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.695 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.695 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.696 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.696 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.697 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:08.890 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:08.892 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:08.895 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.895 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.895 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.895 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.896 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:08.896 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:08.898 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:09.371 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:09.372 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:09.376 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:09.376 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:09.376 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:09.376 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:09.376 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:09.376 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:09.380 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:09.568935 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:09.569239 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:10.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:10.441545 30028 raft_consensus.cc:482] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:45:10.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:10.442008 30028 raft_consensus.cc:504] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:10.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:10.443475 30028 leader_election.cc:288] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d102020153984228a0194a28ea6e894b (127.28.199.126:34497), e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813)
19:45:10.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:10.451620 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d102020153984228a0194a28ea6e894b (127.28.199.126:34497): Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111)
19:45:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:10.452651 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813): Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:45:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:10.453001 29874 leader_election.cc:302] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db4f06f81a0f40f6a51354aa78aac6b8; no voters: d102020153984228a0194a28ea6e894b, e653af98c80247fbae18a0fe8ce38645
19:45:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:10.453769 30028 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:45:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:10.560407 29851 master_runner.cc:186] Error getting master registration for 127.28.199.126:34497: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111),
19:45:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:10.560823 29851 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:45:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:10.572362 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:10.572702 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:10.751 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:10.753 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:10.755 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:10.755 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:10.755 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:10.755 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:10.755 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:10.755 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:10.757 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:11.575789 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:11.576122 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:12.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:12.579833 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:12.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:12.580133 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:13.111 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:13.112 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:13.115 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:13.115 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:13.115 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:13.116 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:13.116 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:13.116 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:13.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:13.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:13.506814 30034 raft_consensus.cc:482] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:45:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:13.507323 30034 raft_consensus.cc:504] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:13.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:13.509109 30034 leader_election.cc:288] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d102020153984228a0194a28ea6e894b (127.28.199.126:34497), e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813)
19:45:13.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:13.512130 29874 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111) [suppressed 5 similar messages]
19:45:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:13.515811 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d102020153984228a0194a28ea6e894b (127.28.199.126:34497): Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111)
19:45:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:13.516471 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813): Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:45:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:13.516747 29874 leader_election.cc:302] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db4f06f81a0f40f6a51354aa78aac6b8; no voters: d102020153984228a0194a28ea6e894b, e653af98c80247fbae18a0fe8ce38645
19:45:13.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:13.517619 30034 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:45:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:13.583402 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:13.583673 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:14.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:14.587678 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:14.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:14.588042 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:15 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045515, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:45:15.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:15.214169 29867 init.cc:295] Successfully reacquired a new kerberos TGT
19:45:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:15.564590 29851 master_runner.cc:186] Error getting master registration for 127.28.199.126:34497: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111),
19:45:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:15.564988 29851 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:45:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:15.590973 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:15.591282 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:15.891 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:15.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:15.895 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:15.895 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:15.895 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:15.896 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:15.896 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:15.896 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:15.897 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:16.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:16.147863 29164 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 29019
19:45:16.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:16.148682 29164 debug-util.cc:398] Leaking SignalData structure 0x7b08000af340 after lost signal to thread 29167
19:45:16.291 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:16.293 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:16.294 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:16.295 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:16.295 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:16.296 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:16.296 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:16.296 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:16.297 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:16.594465 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:16.594813 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:17.595121 30047 raft_consensus.cc:482] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:45:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:17.595811 30047 raft_consensus.cc:504] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:17.598577 30047 leader_election.cc:288] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d102020153984228a0194a28ea6e894b (127.28.199.126:34497), e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813)
19:45:17.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:17.607348 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d102020153984228a0194a28ea6e894b (127.28.199.126:34497): Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111)
19:45:17.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:17.607666 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:17.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:17.607978 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:17.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:17.608074 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813): Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:45:17.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:17.608685 29874 leader_election.cc:302] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db4f06f81a0f40f6a51354aa78aac6b8; no voters: d102020153984228a0194a28ea6e894b, e653af98c80247fbae18a0fe8ce38645
19:45:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:17.610002 30047 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:45:18.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:18.611202 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:18.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:18.611483 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:19.071 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:19.073 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:19.075 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:19.075 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:19.075 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:19.075 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:19.076 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:19.076 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:19.077 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:19.614491 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:19.614826 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:20.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:20.568181 29851 master_runner.cc:186] Error getting master registration for 127.28.199.126:34497: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111),
19:45:20.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:20.568544 29851 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:45:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:20.618165 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:20.618497 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:21.621721 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:21.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:21.622114 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:22.131 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:22.134 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:22.134 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:22.134 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:22.134 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:22.136 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:22.136 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:22.137 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:22.137 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:22.625491 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:22.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:22.625890 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:23.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:23.629230 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:23.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:23.629591 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:23.760648 30057 raft_consensus.cc:482] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:45:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:23.761128 30057 raft_consensus.cc:504] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:23.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:23.763059 30057 leader_election.cc:288] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d102020153984228a0194a28ea6e894b (127.28.199.126:34497), e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813)
19:45:23.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:23.765754 29874 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111) [suppressed 3 similar messages]
19:45:23.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:23.769232 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d102020153984228a0194a28ea6e894b (127.28.199.126:34497): Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111)
19:45:23.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:23.769829 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813): Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:45:23.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:23.770262 29874 leader_election.cc:302] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db4f06f81a0f40f6a51354aa78aac6b8; no voters: d102020153984228a0194a28ea6e894b, e653af98c80247fbae18a0fe8ce38645
19:45:23.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:23.770959 30057 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:45:24.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:24.232468 29164 debug-util.cc:398] Leaking SignalData structure 0x7b08000af580 after lost signal to thread 29019
19:45:24.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:24.233373 29164 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcfc0 after lost signal to thread 29167
19:45:24.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:24.633718 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:24.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:24.634111 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:25.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:25 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045525, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:45:25.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:25.217176 29867 init.cc:295] Successfully reacquired a new kerberos TGT
19:45:25.391 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:25.393 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:25.395 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:25.395 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:25.395 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:25.395 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:25.395 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:25.395 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:25.398 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:25.572137 29851 master_runner.cc:186] Error getting master registration for 127.28.199.126:34497: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111),
19:45:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:25.572609 29851 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:45:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:25.637426 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:25.637739 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:26.640697 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:26.640976 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:27.644081 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:27.644397 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:28.351 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:28.352 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:28.354 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:28.354 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:28.355 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:28.355 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:28.355 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:28.355 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:28.356 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:28.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:28.647387 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:28.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:28.647687 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:29.651098 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:29.651371 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:30.572657 30076 raft_consensus.cc:482] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:45:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:30.573443 30076 raft_consensus.cc:504] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d102020153984228a0194a28ea6e894b" member_type: VOTER last_known_addr { host: "127.28.199.126" port: 34497 } } peers { permanent_uuid: "e653af98c80247fbae18a0fe8ce38645" member_type: VOTER last_known_addr { host: "127.28.199.125" port: 46813 } } peers { permanent_uuid: "db4f06f81a0f40f6a51354aa78aac6b8" member_type: VOTER last_known_addr { host: "127.28.199.124" port: 34413 } }
19:45:30.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:30.575687 30076 leader_election.cc:288] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d102020153984228a0194a28ea6e894b (127.28.199.126:34497), e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813)
19:45:30.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:30.576557 29874 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111) [suppressed 1 similar messages]
19:45:30.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:30.577786 29851 master_runner.cc:186] Error getting master registration for 127.28.199.126:34497: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111),
19:45:30.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:30.578215 29851 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:45:30.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:30.580379 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d102020153984228a0194a28ea6e894b (127.28.199.126:34497): Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111)
19:45:30.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:30.580924 29874 leader_election.cc:334] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e653af98c80247fbae18a0fe8ce38645 (127.28.199.125:46813): Network error: Client connection negotiation failed: client connection to 127.28.199.125:46813: connect: Connection refused (error 111)
19:45:30.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:30.581228 29874 leader_election.cc:302] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db4f06f81a0f40f6a51354aa78aac6b8; no voters: d102020153984228a0194a28ea6e894b, e653af98c80247fbae18a0fe8ce38645
19:45:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:30.581784 30076 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:45:30.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:30.654448 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:30.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:30.654716 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:31.251 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:31.253 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:31.254 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:31.254 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:31.254 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:31.254 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:31.254 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:31.254 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:31.257 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:31.657788 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:31.658066 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:32.291 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:32.292 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:32.293 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:32.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:32.293 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:32.294 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:32.294 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:32.294 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:32.296 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:32.661319 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:32.661639 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:33.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:33.664688 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:33.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:33.664978 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:34.670681 29941 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db4f06f81a0f40f6a51354aa78aac6b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:45:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 19:45:34.671274 29941 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:45:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 19:45:35 dist-test-slave-vxj2 krb5kdc[29481](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786045535, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.199.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:45:35.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:35.222028 29867 init.cc:295] Successfully reacquired a new kerberos TGT
19:45:35.572 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.125:46813(127.28.199.125:46813)] connecting to peer
19:45:35.574 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.199.126:34497(127.28.199.126:34497)] connecting to peer
19:45:35.575 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:35.575 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.125:46813(127.28.199.125:46813)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:35.576 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.125:46813: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813
19:45:35.579 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:35.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.199.126:34497(127.28.199.126:34497)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:35.581 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.199.126:34497: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
19:45:35.584 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497; will retry
19:45:35.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:35.588909 29851 master_runner.cc:186] Error getting master registration for 127.28.199.126:34497: Network error: Client connection negotiation failed: client connection to 127.28.199.126:34497: connect: Connection refused (error 111),
19:45:35.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:35.589788 29851 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:45:35.598 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28056), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [151ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [216ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [219ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [653ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [658ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [673ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [685ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [686ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [689ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [712ms] refreshing cache from master, [713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [718ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [719ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [725ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [732ms] refreshing cache from master, [732ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [738ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [741ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [742ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [772ms] refreshing cache from master, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [777ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [778ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [780ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [781ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [792ms] refreshing cache from master, [793ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [798ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [798ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [799ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [832ms] refreshing cache from master, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [839ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [840ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [842ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [912ms] refreshing cache from master, [913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [917ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [918ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [920ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [921ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1152ms] refreshing cache from master, [1152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1157ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1157ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1160ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1352ms] refreshing cache from master, [1352ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1354ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1357ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1357ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1360ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1832ms] refreshing cache from master, [1832ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1841ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1842ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [3212ms] refreshing cache from master, [3213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [3213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [3215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [3217ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [3217ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [3219ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [3219ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [5572ms] refreshing cache from master, [5572ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [5573ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [5574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [5577ms] too many traces: truncated at 100 traces, deferred=Deferred@1946024300(state=PENDING, result=null, callback=(continuation of Deferred@1850919905 after retry RPC after error@984481877) -> (continuation of Deferred@1499872612 after retry RPC after error@1171032509) -> (continuation of Deferred@1027165990 after retry RPC after error@2002738300) -> (continuation of Deferred@523944686 after retry RPC after error@1050292693) -> (continuation of Deferred@1236080991 after retry RPC after error@1151074474) -> (continuation of Deferred@234395493 after retry RPC after error@1652759446) -> (continuation of Deferred@1128632696 after retry RPC after error@1735934186) -> (continuation of Deferred@787656927 after retry RPC after error@1423397180) -> (continuation of Deferred@1006032015 after retry RPC after error@506248203) -> (continuation of Deferred@627282777 after retry RPC after error@1263757432) -> (continuation of Deferred@1183616817 after retry RPC after error@1035710223) -> (continuation of Deferred@83565242 after retry RPC after error@1421583093) -> (continuation of Deferred@1986812500 after retry RPC after error@382681153) -> (continuation of Deferred@1245554408 after retry RPC after error@1125652816) -> (continuation of Deferred@1471305117 after retry RPC after error@991156834) -> (continuation of Deferred@473837838 after retry RPC after error@527399232) -> (continuation of Deferred@1521797517 after retry RPC after error@2144454771) -> (continuation of Deferred@311042246 after retry RPC after error@1993080494) -> (continuation of Deferred@181385563 after retry RPC after error@2066512417) -> (continuation of Deferred@954460562 after retry RPC after error@292839086) -> (continuation of Deferred@1711679674 after retry RPC after error@2128853801), errback=(continuation of Deferred@1850919905 after retry RPC after error@984481877) -> (continuation of Deferred@1499872612 after retry RPC after error@1171032509) -> (continuation of Deferred@1027165990 after retry RPC after error@2002738300) -> (continuation of Deferred@523944686 after retry RPC after error@1050292693) -> (continuation of Deferred@1236080991 after retry RPC after error@1151074474) -> (continuation of Deferred@234395493 after retry RPC after error@1652759446) -> (continuation of Deferred@1128632696 after retry RPC after error@1735934186) -> (continuation of Deferred@787656927 after retry RPC after error@1423397180) -> (continuation of Deferred@1006032015 after retry RPC after error@506248203) -> (continuation of Deferred@627282777 after retry RPC after error@1263757432) -> (continuation of Deferred@1183616817 after retry RPC after error@1035710223) -> (continuation of Deferred@83565242 after retry RPC after error@1421583093) -> (continuation of Deferred@1986812500 after retry RPC after error@382681153) -> (continuation of Deferred@1245554408 after retry RPC after error@1125652816) -> (continuation of Deferred@1471305117 after retry RPC after error@991156834) -> (continuation of Deferred@473837838 after retry RPC after error@527399232) -> (continuation of Deferred@1521797517 after retry RPC after error@2144454771) -> (continuation of Deferred@311042246 after retry RPC after error@1993080494) -> (continuation of Deferred@181385563 after retry RPC after error@2066512417) -> (continuation of Deferred@954460562 after retry RPC after error@292839086) -> (continuation of Deferred@1711679674 after retry RPC after error@2128853801)))
19:45:35.610 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.199.126:34497(127.28.199.126:34497)] disconnecting while in state TERMINATED
19:45:35.613 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.199.124:34413(127.28.199.124:34413)] disconnecting while in state READY
19:45:35.613 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.199.125:46813(127.28.199.125:46813)] disconnecting while in state TERMINATED
19:45:35.623 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:45:35.628 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.199.124:34413(127.28.199.124:34413)] handling channelInactive
19:45:35.628 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:45:35.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.199.124:34413(127.28.199.124:34413)] cleaning up while in state READY due to: connection closed
19:45:35.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:35.637701 29469 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu with pid 29851
19:45:36.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T19:45:36Z chronyd exiting
19:45:38.023 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:45:38.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:38.025336 28720 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu with pid 29009
19:45:38.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:38.057861 28720 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu with pid 29172
19:45:38.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:38.093822 28720 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu with pid 29326
19:45:38.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:38.125460 28720 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu with pid 28732
19:45:38.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:38.163319 28720 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu with pid 28803
19:45:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 19:45:38.198104 28720 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBYv0IG/build/tsan/bin/kudu with pid 28881
19:45:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T19:45:39Z chronyd exiting
19:45:40.421 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28056), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [151ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [216ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [219ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [653ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [658ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [673ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [685ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [686ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [689ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [712ms] refreshing cache from master, [713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [718ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [719ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [725ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [732ms] refreshing cache from master, [732ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [738ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [741ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [742ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [772ms] refreshing cache from master, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [777ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [778ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [780ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [781ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [792ms] refreshing cache from master, [793ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [798ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [798ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [799ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [832ms] refreshing cache from master, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [839ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [840ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [842ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [912ms] refreshing cache from master, [913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [917ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [918ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [920ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [921ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1152ms] refreshing cache from master, [1152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1157ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1157ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1160ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1352ms] refreshing cache from master, [1352ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1354ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1357ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1357ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1360ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1832ms] refreshing cache from master, [1832ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1841ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1842ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [3212ms] refreshing cache from master, [3213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [3213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [3215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [3217ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [3217ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [3219ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [3219ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [5572ms] refreshing cache from master, [5572ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [5573ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [5574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [5577ms] too many traces: truncated at 100 traces, deferred=Deferred@1946024300(state=PENDING, result=null, callback=(continuation of Deferred@1850919905 after retry RPC after error@984481877) -> (continuation of Deferred@1499872612 after retry RPC after error@1171032509) -> (continuation of Deferred@1027165990 after retry RPC after error@2002738300) -> (continuation of Deferred@523944686 after retry RPC after error@1050292693) -> (continuation of Deferred@1236080991 after retry RPC after error@1151074474) -> (continuation of Deferred@234395493 after retry RPC after error@1652759446) -> (continuation of Deferred@1128632696 after retry RPC after error@1735934186) -> (continuation of Deferred@787656927 after retry RPC after error@1423397180) -> (continuation of Deferred@1006032015 after retry RPC after error@506248203) -> (continuation of Deferred@627282777 after retry RPC after error@1263757432) -> (continuation of Deferred@1183616817 after retry RPC after error@1035710223) -> (continuation of Deferred@83565242 after retry RPC after error@1421583093) -> (continuation of Deferred@1986812500 after retry RPC after error@382681153) -> (continuation of Deferred@1245554408 after retry RPC after error@1125652816) -> (continuation of Deferred@1471305117 after retry RPC after error@991156834) -> (continuation of Deferred@473837838 after retry RPC after error@527399232) -> (continuation of Deferred@1521797517 after retry RPC after error@2144454771) -> (continuation of Deferred@311042246 after retry RPC after error@1993080494) -> (continuation of Deferred@181385563 after retry RPC after error@2066512417) -> (continuation of Deferred@954460562 after retry RPC after error@292839086) -> (continuation of Deferred@1711679674 after retry RPC after error@2128853801), errback=(continuation of Deferred@1850919905 after retry RPC after error@984481877) -> (continuation of Deferred@1499872612 after retry RPC after error@1171032509) -> (continuation of Deferred@1027165990 after retry RPC after error@2002738300) -> (continuation of Deferred@523944686 after retry RPC after error@1050292693) -> (continuation of Deferred@1236080991 after retry RPC after error@1151074474) -> (continuation of Deferred@234395493 after retry RPC after error@1652759446) -> (continuation of Deferred@1128632696 after retry RPC after error@1735934186) -> (continuation of Deferred@787656927 after retry RPC after error@1423397180) -> (continuation of Deferred@1006032015 after retry RPC after error@506248203) -> (continuation of Deferred@627282777 after retry RPC after error@1263757432) -> (continuation of Deferred@1183616817 after retry RPC after error@1035710223) -> (continuation of Deferred@83565242 after retry RPC after error@1421583093) -> (continuation of Deferred@1986812500 after retry RPC after error@382681153) -> (continuation of Deferred@1245554408 after retry RPC after error@1125652816) -> (continuation of Deferred@1471305117 after retry RPC after error@991156834) -> (continuation of Deferred@473837838 after retry RPC after error@527399232) -> (continuation of Deferred@1521797517 after retry RPC after error@2144454771) -> (continuation of Deferred@311042246 after retry RPC after error@1993080494) -> (continuation of Deferred@181385563 after retry RPC after error@2066512417) -> (continuation of Deferred@954460562 after retry RPC after error@292839086) -> (continuation of Deferred@1711679674 after retry RPC after error@2128853801)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.28.199.126:34497
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28056), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [151ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [216ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [219ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [653ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [658ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [673ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [685ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [686ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [689ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [712ms] refreshing cache from master, [713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [718ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [719ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [725ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [732ms] refreshing cache from master, [732ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [738ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [741ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [742ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [772ms] refreshing cache from master, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [777ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [778ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [780ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [781ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [792ms] refreshing cache from master, [793ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [798ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [798ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [799ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [832ms] refreshing cache from master, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [839ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [840ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [842ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [912ms] refreshing cache from master, [913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [917ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [918ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [920ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [921ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1152ms] refreshing cache from master, [1152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1157ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1157ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1160ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1352ms] refreshing cache from master, [1352ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1354ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1357ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1357ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1360ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1832ms] refreshing cache from master, [1832ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1841ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1842ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [3212ms] refreshing cache from master, [3213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [3213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [3215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [3217ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [3217ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [3219ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [3219ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [5572ms] refreshing cache from master, [5572ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [5573ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [5574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [5577ms] too many traces: truncated at 100 traces, deferred=Deferred@1946024300(state=PENDING, result=null, callback=(continuation of Deferred@1850919905 after retry RPC after error@984481877) -> (continuation of Deferred@1499872612 after retry RPC after error@1171032509) -> (continuation of Deferred@1027165990 after retry RPC after error@2002738300) -> (continuation of Deferred@523944686 after retry RPC after error@1050292693) -> (continuation of Deferred@1236080991 after retry RPC after error@1151074474) -> (continuation of Deferred@234395493 after retry RPC after error@1652759446) -> (continuation of Deferred@1128632696 after retry RPC after error@1735934186) -> (continuation of Deferred@787656927 after retry RPC after error@1423397180) -> (continuation of Deferred@1006032015 after retry RPC after error@506248203) -> (continuation of Deferred@627282777 after retry RPC after error@1263757432) -> (continuation of Deferred@1183616817 after retry RPC after error@1035710223) -> (continuation of Deferred@83565242 after retry RPC after error@1421583093) -> (continuation of Deferred@1986812500 after retry RPC after error@382681153) -> (continuation of Deferred@1245554408 after retry RPC after error@1125652816) -> (continuation of Deferred@1471305117 after retry RPC after error@991156834) -> (continuation of Deferred@473837838 after retry RPC after error@527399232) -> (continuation of Deferred@1521797517 after retry RPC after error@2144454771) -> (continuation of Deferred@311042246 after retry RPC after error@1993080494) -> (continuation of Deferred@181385563 after retry RPC after error@2066512417) -> (continuation of Deferred@954460562 after retry RPC after error@292839086) -> (continuation of Deferred@1711679674 after retry RPC after error@2128853801), errback=(continuation of Deferred@1850919905 after retry RPC after error@984481877) -> (continuation of Deferred@1499872612 after retry RPC after error@1171032509) -> (continuation of Deferred@1027165990 after retry RPC after error@2002738300) -> (continuation of Deferred@523944686 after retry RPC after error@1050292693) -> (continuation of Deferred@1236080991 after retry RPC after error@1151074474) -> (continuation of Deferred@234395493 after retry RPC after error@1652759446) -> (continuation of Deferred@1128632696 after retry RPC after error@1735934186) -> (continuation of Deferred@787656927 after retry RPC after error@1423397180) -> (continuation of Deferred@1006032015 after retry RPC after error@506248203) -> (continuation of Deferred@627282777 after retry RPC after error@1263757432) -> (continuation of Deferred@1183616817 after retry RPC after error@1035710223) -> (continuation of Deferred@83565242 after retry RPC after error@1421583093) -> (continuation of Deferred@1986812500 after retry RPC after error@382681153) -> (continuation of Deferred@1245554408 after retry RPC after error@1125652816) -> (continuation of Deferred@1471305117 after retry RPC after error@991156834) -> (continuation of Deferred@473837838 after retry RPC after error@527399232) -> (continuation of Deferred@1521797517 after retry RPC after error@2144454771) -> (continuation of Deferred@311042246 after retry RPC after error@1993080494) -> (continuation of Deferred@181385563 after retry RPC after error@2066512417) -> (continuation of Deferred@954460562 after retry RPC after error@292839086) -> (continuation of Deferred@1711679674 after retry RPC after error@2128853801)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.28.199.126:34497
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
19:45:41.419 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
19:45:41.419 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28056), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [151ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [216ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [219ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [653ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [658ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [673ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [685ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [686ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [689ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [712ms] refreshing cache from master, [713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [718ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [719ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [725ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [732ms] refreshing cache from master, [732ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [738ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [741ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [742ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [772ms] refreshing cache from master, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [777ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [778ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [780ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [781ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [792ms] refreshing cache from master, [793ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [798ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [798ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [799ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [832ms] refreshing cache from master, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [839ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [840ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [842ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [912ms] refreshing cache from master, [913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [917ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [918ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [920ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [921ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1152ms] refreshing cache from master, [1152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1157ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1157ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1160ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1352ms] refreshing cache from master, [1352ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1354ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1357ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1357ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1360ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1832ms] refreshing cache from master, [1832ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1841ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1842ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [3212ms] refreshing cache from master, [3213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [3213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [3215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [3217ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [3217ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [3219ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [3219ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [5572ms] refreshing cache from master, [5572ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [5573ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [5574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [5577ms] too many traces: truncated at 100 traces, deferred=Deferred@1946024300(state=PENDING, result=null, callback=(continuation of Deferred@1850919905 after retry RPC after error@984481877) -> (continuation of Deferred@1499872612 after retry RPC after error@1171032509) -> (continuation of Deferred@1027165990 after retry RPC after error@2002738300) -> (continuation of Deferred@523944686 after retry RPC after error@1050292693) -> (continuation of Deferred@1236080991 after retry RPC after error@1151074474) -> (continuation of Deferred@234395493 after retry RPC after error@1652759446) -> (continuation of Deferred@1128632696 after retry RPC after error@1735934186) -> (continuation of Deferred@787656927 after retry RPC after error@1423397180) -> (continuation of Deferred@1006032015 after retry RPC after error@506248203) -> (continuation of Deferred@627282777 after retry RPC after error@1263757432) -> (continuation of Deferred@1183616817 after retry RPC after error@1035710223) -> (continuation of Deferred@83565242 after retry RPC after error@1421583093) -> (continuation of Deferred@1986812500 after retry RPC after error@382681153) -> (continuation of Deferred@1245554408 after retry RPC after error@1125652816) -> (continuation of Deferred@1471305117 after retry RPC after error@991156834) -> (continuation of Deferred@473837838 after retry RPC after error@527399232) -> (continuation of Deferred@1521797517 after retry RPC after error@2144454771) -> (continuation of Deferred@311042246 after retry RPC after error@1993080494) -> (continuation of Deferred@181385563 after retry RPC after error@2066512417) -> (continuation of Deferred@954460562 after retry RPC after error@292839086) -> (continuation of Deferred@1711679674 after retry RPC after error@2128853801), errback=(continuation of Deferred@1850919905 after retry RPC after error@984481877) -> (continuation of Deferred@1499872612 after retry RPC after error@1171032509) -> (continuation of Deferred@1027165990 after retry RPC after error@2002738300) -> (continuation of Deferred@523944686 after retry RPC after error@1050292693) -> (continuation of Deferred@1236080991 after retry RPC after error@1151074474) -> (continuation of Deferred@234395493 after retry RPC after error@1652759446) -> (continuation of Deferred@1128632696 after retry RPC after error@1735934186) -> (continuation of Deferred@787656927 after retry RPC after error@1423397180) -> (continuation of Deferred@1006032015 after retry RPC after error@506248203) -> (continuation of Deferred@627282777 after retry RPC after error@1263757432) -> (continuation of Deferred@1183616817 after retry RPC after error@1035710223) -> (continuation of Deferred@83565242 after retry RPC after error@1421583093) -> (continuation of Deferred@1986812500 after retry RPC after error@382681153) -> (continuation of Deferred@1245554408 after retry RPC after error@1125652816) -> (continuation of Deferred@1471305117 after retry RPC after error@991156834) -> (continuation of Deferred@473837838 after retry RPC after error@527399232) -> (continuation of Deferred@1521797517 after retry RPC after error@2144454771) -> (continuation of Deferred@311042246 after retry RPC after error@1993080494) -> (continuation of Deferred@181385563 after retry RPC after error@2066512417) -> (continuation of Deferred@954460562 after retry RPC after error@292839086) -> (continuation of Deferred@1711679674 after retry RPC after error@2128853801)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.28.199.126:34497
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28056), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [151ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [153ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [216ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [219ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [653ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [658ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [673ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [685ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [686ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [689ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [712ms] refreshing cache from master, [713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [718ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [719ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [725ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [732ms] refreshing cache from master, [732ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [738ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [741ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [742ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [772ms] refreshing cache from master, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [777ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [778ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [780ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [781ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [792ms] refreshing cache from master, [793ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [798ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [798ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [799ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [832ms] refreshing cache from master, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [839ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [840ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [842ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [912ms] refreshing cache from master, [913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [917ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [918ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [920ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [921ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1152ms] refreshing cache from master, [1152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1157ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1157ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1160ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1352ms] refreshing cache from master, [1352ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1354ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1357ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1357ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1360ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1832ms] refreshing cache from master, [1832ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [1834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [1834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [1838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [1838ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [1841ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [1842ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [3212ms] refreshing cache from master, [3213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [3213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [3215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [3217ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.126:34497: Network error: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497, [3217ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.125:46813: Network error: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [3219ms] Sub RPC ConnectToMaster: received response from server master-127.28.199.124:34413: OK, [3219ms] delaying RPC due to: Service unavailable: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813, [5572ms] refreshing cache from master, [5572ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.125:46813, [5573ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.124:34413, [5574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.199.126:34497, [5577ms] too many traces: truncated at 100 traces, deferred=Deferred@1946024300(state=PENDING, result=null, callback=(continuation of Deferred@1850919905 after retry RPC after error@984481877) -> (continuation of Deferred@1499872612 after retry RPC after error@1171032509) -> (continuation of Deferred@1027165990 after retry RPC after error@2002738300) -> (continuation of Deferred@523944686 after retry RPC after error@1050292693) -> (continuation of Deferred@1236080991 after retry RPC after error@1151074474) -> (continuation of Deferred@234395493 after retry RPC after error@1652759446) -> (continuation of Deferred@1128632696 after retry RPC after error@1735934186) -> (continuation of Deferred@787656927 after retry RPC after error@1423397180) -> (continuation of Deferred@1006032015 after retry RPC after error@506248203) -> (continuation of Deferred@627282777 after retry RPC after error@1263757432) -> (continuation of Deferred@1183616817 after retry RPC after error@1035710223) -> (continuation of Deferred@83565242 after retry RPC after error@1421583093) -> (continuation of Deferred@1986812500 after retry RPC after error@382681153) -> (continuation of Deferred@1245554408 after retry RPC after error@1125652816) -> (continuation of Deferred@1471305117 after retry RPC after error@991156834) -> (continuation of Deferred@473837838 after retry RPC after error@527399232) -> (continuation of Deferred@1521797517 after retry RPC after error@2144454771) -> (continuation of Deferred@311042246 after retry RPC after error@1993080494) -> (continuation of Deferred@181385563 after retry RPC after error@2066512417) -> (continuation of Deferred@954460562 after retry RPC after error@292839086) -> (continuation of Deferred@1711679674 after retry RPC after error@2128853801), errback=(continuation of Deferred@1850919905 after retry RPC after error@984481877) -> (continuation of Deferred@1499872612 after retry RPC after error@1171032509) -> (continuation of Deferred@1027165990 after retry RPC after error@2002738300) -> (continuation of Deferred@523944686 after retry RPC after error@1050292693) -> (continuation of Deferred@1236080991 after retry RPC after error@1151074474) -> (continuation of Deferred@234395493 after retry RPC after error@1652759446) -> (continuation of Deferred@1128632696 after retry RPC after error@1735934186) -> (continuation of Deferred@787656927 after retry RPC after error@1423397180) -> (continuation of Deferred@1006032015 after retry RPC after error@506248203) -> (continuation of Deferred@627282777 after retry RPC after error@1263757432) -> (continuation of Deferred@1183616817 after retry RPC after error@1035710223) -> (continuation of Deferred@83565242 after retry RPC after error@1421583093) -> (continuation of Deferred@1986812500 after retry RPC after error@382681153) -> (continuation of Deferred@1245554408 after retry RPC after error@1125652816) -> (continuation of Deferred@1471305117 after retry RPC after error@991156834) -> (continuation of Deferred@473837838 after retry RPC after error@527399232) -> (continuation of Deferred@1521797517 after retry RPC after error@2144454771) -> (continuation of Deferred@311042246 after retry RPC after error@1993080494) -> (continuation of Deferred@181385563 after retry RPC after error@2066512417) -> (continuation of Deferred@954460562 after retry RPC after error@292839086) -> (continuation of Deferred@1711679674 after retry RPC after error@2128853801)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.28.199.125:46813,127.28.199.124:34413,127.28.199.126:34497) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.125:46813(127.28.199.125:46813): Connection refused: /127.28.199.125:46813,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.199.126:34497(127.28.199.126:34497): Connection refused: /127.28.199.126:34497
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.28.199.126:34497
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)