20:13:22.743 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:13:22.757 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasknE2NB6/test-tmp
20:13:22.760 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:13:22.764 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasknE2NB6/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:13:23.017 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:13:23.328 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:13:23.329 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:13:23.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:23.337632 6611 mini_oidc.cc:152] Starting JWKS server
20:13:23.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:23.345353 6611 webserver.cc:460] Webserver started at http://127.0.0.1:37189/ using document root <none> and password file <none>
20:13:23.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:23.346082 6611 mini_oidc.cc:189] Starting OIDC Discovery server
20:13:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:23.348492 6611 webserver.cc:460] Webserver started at http://127.0.0.1:34715/ using document root <none> and password file <none>
20:13:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T20:13:23Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:13:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T20:13:23Z Disabled control of system clock
20:13:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:23.418591 6611 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/wal
20:13:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/data
20:13:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/logs
20:13:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/data/info.pb
20:13:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:13:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.254:40471
20:13:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.254
20:13:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:23.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:23.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37189/jwks.json with env {}
20:13:23.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.781802 6629 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:23.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.783007 6629 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:23.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.783780 6629 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.820240 6629 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:13:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.820609 6629 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.820829 6629 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:13:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.821022 6629 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:13:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.821209 6629 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:13:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.821346 6629 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37189/jwks.json
20:13:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:23.863554 6629 master_runner.cc:386] Master server non-default flags:
20:13:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/data
20:13:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/wal
20:13:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.254:40471
20:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37189/jwks.json
20:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/data/info.pb
20:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.254
20:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/logs
20:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:23.866250 6629 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:23.869603 6629 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:23.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.901497 6638 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.901575 6637 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:23.912220 6641 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:25.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:23.903391 6629 server_base.cc:1034] running on GCE node
20:13:25.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.215593 6629 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.220378 6629 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.221997 6629 hybrid_clock.cc:648] HybridClock initialized: now 1786133605221938 us; error 122 us; skew 500 ppm
20:13:25.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.223284 6629 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:25.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.236650 6629 webserver.cc:460] Webserver started at http://127.6.116.254:33013/ using document root <none> and password file <none>
20:13:25.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.238080 6629 fs_manager.cc:359] Metadata directory not provided
20:13:25.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.238314 6629 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:25.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.238857 6629 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:13:25.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.253554 6629 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/data/instance:
20:13:25.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "92ea8f927bc44b4281ae8b0dbe29df14"
20:13:25.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:25 on dist-test-slave-kvfs"
20:13:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.255789 6629 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/wal/instance:
20:13:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "92ea8f927bc44b4281ae8b0dbe29df14"
20:13:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:25 on dist-test-slave-kvfs"
20:13:25.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.266186 6629 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
20:13:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.273293 6648 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:25.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.274869 6629 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
20:13:25.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.275314 6629 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/wal
20:13:25.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "92ea8f927bc44b4281ae8b0dbe29df14"
20:13:25.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:25 on dist-test-slave-kvfs"
20:13:25.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.275691 6629 fs_report.cc:383] FS layout report
20:13:25.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:25.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/wal
20:13:25.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/wal
20:13:25.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/data/data
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.319144 6629 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.321465 6629 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.322173 6629 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.430420 6629 rpc_server.cc:307] RPC server started. Bound to: 127.6.116.254:40471
20:13:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.430511 6720 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.116.254:40471 every 8 connection(s)
20:13:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.434799 6629 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/data/info.pb
20:13:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.444111 6611 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 6629
20:13:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.445158 6611 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-0/wal/instance
20:13:25.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.448416 6721 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.453486 6611 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/wal
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/data
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/logs
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/data/info.pb
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.253:38251
20:13:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.253
20:13:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:25.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:25.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:25.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:25.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37189/jwks.json with env {}
20:13:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.471621 6721 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } has no permanent_uuid. Determining permanent_uuid...
20:13:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.522819 6721 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } has no permanent_uuid. Determining permanent_uuid...
20:13:25.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.526290 6649 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.116.253:38251: connect: Connection refused (error 111)
20:13:25.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.532950 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.253:38251: Network error: Client connection negotiation failed: client connection to 127.6.116.253:38251: connect: Connection refused (error 111)
20:13:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.582847 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } attempt: 1
20:13:25.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.588485 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.253:38251: Network error: Client connection negotiation failed: client connection to 127.6.116.253:38251: connect: Connection refused (error 111)
20:13:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.657016 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } attempt: 2
20:13:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.661979 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.253:38251: Network error: Client connection negotiation failed: client connection to 127.6.116.253:38251: connect: Connection refused (error 111)
20:13:25.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.753500 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } attempt: 3
20:13:25.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.758599 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.253:38251: Network error: Client connection negotiation failed: client connection to 127.6.116.253:38251: connect: Connection refused (error 111)
20:13:25.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.861937 6723 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:25.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.862710 6723 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:25.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.863322 6723 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.899471 6723 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:13:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.899818 6723 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.900174 6723 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:13:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.900512 6723 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:13:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.900822 6723 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:13:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.901083 6723 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37189/jwks.json
20:13:25.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.902097 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } attempt: 4
20:13:25.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.907085 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.253:38251: Network error: Client connection negotiation failed: client connection to 127.6.116.253:38251: connect: Connection refused (error 111)
20:13:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.942493 6723 master_runner.cc:386] Master server non-default flags:
20:13:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/data
20:13:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/wal
20:13:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.253:38251
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37189/jwks.json
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/data/info.pb
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.253
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:25.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/logs
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.944941 6723 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:25.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:25.948199 6723 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:25.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.971048 6738 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:26.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:26.206624 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } attempt: 5
20:13:26.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:26.212750 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.253:38251: Network error: Client connection negotiation failed: client connection to 127.6.116.253:38251: connect: Connection refused (error 111)
20:13:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:26.760236 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } attempt: 6
20:13:26.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:26.769120 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.253:38251: Network error: Client connection negotiation failed: client connection to 127.6.116.253:38251: connect: Connection refused (error 111)
20:13:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.972508 6740 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:27.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:25.971215 6737 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:27.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.253619 6739 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:13:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.254240 6723 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:13:27.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.258884 6723 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:27.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.263553 6723 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:27.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.265058 6723 hybrid_clock.cc:648] HybridClock initialized: now 1786133607264999 us; error 84 us; skew 500 ppm
20:13:27.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.266053 6723 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:27.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.280853 6723 webserver.cc:460] Webserver started at http://127.6.116.253:36151/ using document root <none> and password file <none>
20:13:27.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.282410 6723 fs_manager.cc:359] Metadata directory not provided
20:13:27.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.282722 6723 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:27.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.283414 6723 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:13:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.289945 6723 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/data/instance:
20:13:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6251c96b8534d898aff862b1c9e3140"
20:13:27.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:27 on dist-test-slave-kvfs"
20:13:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.291610 6723 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/wal/instance:
20:13:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6251c96b8534d898aff862b1c9e3140"
20:13:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:27 on dist-test-slave-kvfs"
20:13:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.301676 6723 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.003s sys 0.005s
20:13:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.309254 6749 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.310766 6723 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.004s
20:13:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.311134 6723 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/wal
20:13:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6251c96b8534d898aff862b1c9e3140"
20:13:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:27 on dist-test-slave-kvfs"
20:13:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.311527 6723 fs_report.cc:383] FS layout report
20:13:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/wal
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/wal
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/data/data
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.379951 6723 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.382153 6723 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.382702 6723 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:27.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.519472 6723 rpc_server.cc:307] RPC server started. Bound to: 127.6.116.253:38251
20:13:27.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.519604 6810 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.116.253:38251 every 8 connection(s)
20:13:27.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.522727 6723 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/data/info.pb
20:13:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.527122 6611 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 6723
20:13:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.527791 6611 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-1/wal/instance
20:13:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.531083 6811 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:13:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.531432 6611 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/wal
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/data
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/logs
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/data/info.pb
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:13:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.252:42795
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.252
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37189/jwks.json with env {}
20:13:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.549185 6811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } has no permanent_uuid. Determining permanent_uuid...
20:13:27.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.579071 6811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } has no permanent_uuid. Determining permanent_uuid...
20:13:27.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.600183 6811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } has no permanent_uuid. Determining permanent_uuid...
20:13:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.602952 6752 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:27.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.608570 6811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.658337 6811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 1
20:13:27.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.663789 6811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:27.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.732314 6811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 2
20:13:27.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.737241 6811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:27.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.828868 6811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 3
20:13:27.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.829921 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } attempt: 7
20:13:27.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.835857 6811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.846825 6721 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } has no permanent_uuid. Determining permanent_uuid...
20:13:27.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.858927 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:27.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.917518 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 1
20:13:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.922817 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.941305 6816 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.942099 6816 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:27.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.942749 6816 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.976972 6816 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:13:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.977357 6816 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.977594 6816 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:13:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.977784 6816 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:13:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.978018 6816 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:13:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.978171 6816 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37189/jwks.json
20:13:27.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:27.979467 6811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 4
20:13:27.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:27.984333 6811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:28.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:28.004359 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 2
20:13:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:28.009366 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:28.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:28.021833 6816 master_runner.cc:386] Master server non-default flags:
20:13:28.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:28.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:28.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/data
20:13:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/wal
20:13:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.252:42795
20:13:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37189/jwks.json
20:13:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/data/info.pb
20:13:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.252
20:13:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/logs
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:28.023836 6816 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:28.026675 6816 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:28.053617 6831 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:28.094945 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 3
20:13:28.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:28.101364 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:28.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:28.241864 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 4
20:13:28.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:28.247291 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:28.283941 6811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 5
20:13:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:28.289897 6811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:28.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:28.530833 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 5
20:13:28.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:28.537014 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:28.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:28.837618 6811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 6
20:13:28.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:28.846531 6811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:29.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.089551 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 6
20:13:29.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:29.098016 6721 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.116.252:42795: Network error: Client connection negotiation failed: client connection to 127.6.116.252:42795: connect: Connection refused (error 111)
20:13:29.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:28.055578 6833 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:29.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:28.053664 6830 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:29.307677 6832 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:13:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.308427 6816 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:13:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.313308 6816 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:29.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.317090 6816 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:29.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.318670 6816 hybrid_clock.cc:648] HybridClock initialized: now 1786133609318610 us; error 85 us; skew 500 ppm
20:13:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.319856 6816 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:29.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.328737 6816 webserver.cc:460] Webserver started at http://127.6.116.252:43097/ using document root <none> and password file <none>
20:13:29.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.330135 6816 fs_manager.cc:359] Metadata directory not provided
20:13:29.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.330403 6816 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:29.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.331087 6816 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:13:29.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.337551 6816 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/data/instance:
20:13:29.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2105a753f124483f9bef5878e5cf8777"
20:13:29.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:29 on dist-test-slave-kvfs"
20:13:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.339164 6816 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/wal/instance:
20:13:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2105a753f124483f9bef5878e5cf8777"
20:13:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:29 on dist-test-slave-kvfs"
20:13:29.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.348508 6816 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.001s
20:13:29.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.355609 6847 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:29.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.357115 6816 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
20:13:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.357532 6816 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/wal
20:13:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2105a753f124483f9bef5878e5cf8777"
20:13:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:29 on dist-test-slave-kvfs"
20:13:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.358040 6816 fs_report.cc:383] FS layout report
20:13:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/wal
20:13:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/wal
20:13:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/data/data
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.405035 6816 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:29.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.407101 6816 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:29.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.407748 6816 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.521344 6816 rpc_server.cc:307] RPC server started. Bound to: 127.6.116.252:42795
20:13:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.521382 6911 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.116.252:42795 every 8 connection(s)
20:13:29.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.526854 6816 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/data/info.pb
20:13:29.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.529287 6611 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 6816
20:13:29.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.529974 6611 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/master-2/wal/instance
20:13:29.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.537832 6912 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:13:29.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.563150 6912 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } has no permanent_uuid. Determining permanent_uuid...
20:13:29.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.597872 6912 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } has no permanent_uuid. Determining permanent_uuid...
20:13:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.612010 6912 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } has no permanent_uuid. Determining permanent_uuid...
20:13:29.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.637899 6912 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777: Bootstrap starting.
20:13:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.649415 6912 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777: Neither blocks nor log segments found. Creating new log.
20:13:29.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.653015 6912 log.cc:824] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777: Log is configured to *not* fsync() on all Append() calls
20:13:29.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.659850 6912 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777: No bootstrap required, opened a new log
20:13:29.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.688649 6912 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } }
20:13:29.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.690022 6912 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:13:29.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.690524 6912 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2105a753f124483f9bef5878e5cf8777, State: Initialized, Role: FOLLOWER
20:13:29.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.692512 6912 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } }
20:13:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.696307 6923 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } } }
20:13:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.697136 6923 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [sys.catalog]: This master's current role is: FOLLOWER
20:13:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.698438 6912 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [sys.catalog]: configured and running, proceeding with master startup.
20:13:29.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:29.745056 6939 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:13:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:29.745489 6939 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:13:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.736528 6816 master_runner.cc:186] Error getting master registration for 127.6.116.254:40471: OK, instance_id { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" instance_seqno: 1786133605401604 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:13:29.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.763882 6816 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:13:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.907248 6811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 7
20:13:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:29.916090 6853 tablet.cc:2369] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:13:29.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.934355 6811 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140: Bootstrap starting.
20:13:29.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.942185 6811 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140: Neither blocks nor log segments found. Creating new log.
20:13:29.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.944690 6811 log.cc:824] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140: Log is configured to *not* fsync() on all Append() calls
20:13:29.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.952163 6811 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140: No bootstrap required, opened a new log
20:13:29.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.990207 6811 raft_consensus.cc:348] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } }
20:13:29.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.991381 6811 raft_consensus.cc:374] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:13:29.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.991776 6811 raft_consensus.cc:729] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a6251c96b8534d898aff862b1c9e3140, State: Initialized, Role: FOLLOWER
20:13:29.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.993006 6811 consensus_queue.cc:260] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } }
20:13:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.996634 6944 sys_catalog.cc:455] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } } }
20:13:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.997797 6944 sys_catalog.cc:458] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [sys.catalog]: This master's current role is: FOLLOWER
20:13:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:29.999205 6811 sys_catalog.cc:564] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [sys.catalog]: configured and running, proceeding with master startup.
20:13:30.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.019603 6944 raft_consensus.cc:482] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:13:30.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.020433 6944 raft_consensus.cc:504] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } }
20:13:30.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.024715 6944 leader_election.cc:288] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 92ea8f927bc44b4281ae8b0dbe29df14 (127.6.116.254:40471), 2105a753f124483f9bef5878e5cf8777 (127.6.116.252:42795)
20:13:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.027201 6878 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a6251c96b8534d898aff862b1c9e3140" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2105a753f124483f9bef5878e5cf8777" is_pre_election: true
20:13:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.028813 6878 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a6251c96b8534d898aff862b1c9e3140 in term 0.
20:13:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.029500 6923 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:13:30.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.030110 6923 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } }
20:13:30.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.030762 6752 leader_election.cc:302] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2105a753f124483f9bef5878e5cf8777, a6251c96b8534d898aff862b1c9e3140; no voters:
20:13:30.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.032219 6944 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:13:30.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.032790 6944 raft_consensus.cc:482] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:13:30.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.033308 6944 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 0 FOLLOWER]: Advancing to term 1
20:13:30.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.042500 6944 raft_consensus.cc:504] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } }
20:13:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.044484 6783 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2105a753f124483f9bef5878e5cf8777" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a6251c96b8534d898aff862b1c9e3140" is_pre_election: true
20:13:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.045972 6783 raft_consensus.cc:2379] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 2105a753f124483f9bef5878e5cf8777 in current term 1: Already voted for candidate a6251c96b8534d898aff862b1c9e3140 in this term.
20:13:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.043330 6923 leader_election.cc:288] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 92ea8f927bc44b4281ae8b0dbe29df14 (127.6.116.254:40471), a6251c96b8534d898aff862b1c9e3140 (127.6.116.253:38251)
20:13:30.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.054455 6944 leader_election.cc:288] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [CANDIDATE]: Term 1 election: Requested vote from peers 92ea8f927bc44b4281ae8b0dbe29df14 (127.6.116.254:40471), 2105a753f124483f9bef5878e5cf8777 (127.6.116.252:42795)
20:13:30.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.063169 6878 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a6251c96b8534d898aff862b1c9e3140" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2105a753f124483f9bef5878e5cf8777"
20:13:30.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.063877 6878 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [term 0 FOLLOWER]: Advancing to term 1
20:13:30.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:30.070057 6960 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:13:30.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:30.070928 6960 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:13:30.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.076573 6878 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a6251c96b8534d898aff862b1c9e3140 in term 1.
20:13:30.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.078495 6752 leader_election.cc:302] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2105a753f124483f9bef5878e5cf8777, a6251c96b8534d898aff862b1c9e3140; no voters:
20:13:30.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.079756 6944 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 1 FOLLOWER]: Leader election won for term 1
20:13:30.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.093050 6944 raft_consensus.cc:686] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [term 1 LEADER]: Becoming Leader. State: Replica: a6251c96b8534d898aff862b1c9e3140, State: Running, Role: LEADER
20:13:30.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.095207 6944 consensus_queue.cc:237] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } }
20:13:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.119261 6962 sys_catalog.cc:455] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a6251c96b8534d898aff862b1c9e3140. Latest consensus state: current_term: 1 leader_uuid: "a6251c96b8534d898aff862b1c9e3140" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } } }
20:13:30.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.126633 6962 sys_catalog.cc:458] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [sys.catalog]: This master's current role is: LEADER
20:13:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.074770 6723 master_runner.cc:186] Error getting master registration for 127.6.116.254:40471: OK, instance_id { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" instance_seqno: 1786133605401604 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:13:30.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.128999 6723 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:13:30.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.131196 6965 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:13:30.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.131572 6721 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } attempt: 7
20:13:30.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:30.143221 6755 tablet.cc:2369] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:13:30.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.143704 6965 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:13:30.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.191829 6721 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14: Bootstrap starting.
20:13:30.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.204094 6721 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14: Neither blocks nor log segments found. Creating new log.
20:13:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.208010 6721 log.cc:824] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14: Log is configured to *not* fsync() on all Append() calls
20:13:30.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.212968 6878 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [term 1 FOLLOWER]: Refusing update from remote peer a6251c96b8534d898aff862b1c9e3140: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:13:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.215304 6944 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:13:30.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.224972 6721 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14: No bootstrap required, opened a new log
20:13:30.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.255019 6923 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a6251c96b8534d898aff862b1c9e3140. Latest consensus state: current_term: 1 leader_uuid: "a6251c96b8534d898aff862b1c9e3140" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } } }
20:13:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.262236 6923 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [sys.catalog]: This master's current role is: FOLLOWER
20:13:30.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.269358 6721 raft_consensus.cc:348] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } }
20:13:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.271096 6721 raft_consensus.cc:374] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:13:30.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.271836 6721 raft_consensus.cc:729] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 92ea8f927bc44b4281ae8b0dbe29df14, State: Initialized, Role: FOLLOWER
20:13:30.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.272461 6968 mvcc.cc:205] Tried to move back new op lower bound from 7316003267337269248 to 7316003266966716416. Current Snapshot: MvccSnapshot[applied={T|T < 7316003267337269248}]
20:13:30.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.273703 6721 consensus_queue.cc:260] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } }
20:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.284752 6962 sys_catalog.cc:455] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a6251c96b8534d898aff862b1c9e3140" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } } }
20:13:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.285746 6962 sys_catalog.cc:458] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [sys.catalog]: This master's current role is: LEADER
20:13:30.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.289567 6923 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a6251c96b8534d898aff862b1c9e3140" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } } }
20:13:30.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.290508 6923 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [sys.catalog]: This master's current role is: FOLLOWER
20:13:30.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.290674 6721 sys_catalog.cc:564] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [sys.catalog]: configured and running, proceeding with master startup.
20:13:30.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.290879 6684 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a6251c96b8534d898aff862b1c9e3140" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" is_pre_election: true
20:13:30.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.288635 6973 sys_catalog.cc:455] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } } }
20:13:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.291971 6682 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a6251c96b8534d898aff862b1c9e3140" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92ea8f927bc44b4281ae8b0dbe29df14"
20:13:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.292651 6683 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2105a753f124483f9bef5878e5cf8777" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" is_pre_election: true
20:13:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.304181 6965 catalog_manager.cc:1347] Generated new cluster ID: 5dfb40bc5595400ca65c3af3b51a1411
20:13:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.291965 6684 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a6251c96b8534d898aff862b1c9e3140 in term 0.
20:13:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.304596 6965 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:13:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.305990 6681 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [term 0 FOLLOWER]: Advancing to term 1
20:13:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.306627 6973 sys_catalog.cc:458] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [sys.catalog]: This master's current role is: FOLLOWER
20:13:30.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.311115 6850 leader_election.cc:302] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2105a753f124483f9bef5878e5cf8777; no voters: 92ea8f927bc44b4281ae8b0dbe29df14, a6251c96b8534d898aff862b1c9e3140
20:13:30.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.312916 6923 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:13:30.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.318816 6681 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [term 1 FOLLOWER]: Refusing update from remote peer a6251c96b8534d898aff862b1c9e3140: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:13:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.328580 6962 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [LEADER]: Connected to new peer: Peer: permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:13:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:30.336404 6655 tablet.cc:2369] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:13:30.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.360584 6965 catalog_manager.cc:1370] Generated new certificate authority record
20:13:30.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.363822 6965 catalog_manager.cc:1504] Loading token signing keys...
20:13:30.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.413410 6965 catalog_manager.cc:5868] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140: Generated new TSK 0
20:13:30.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.415217 6965 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:13:30.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.451166 6611 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/wal
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/data
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/logs
20:13:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/data/info.pb
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.193:0
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.116.193
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.193
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:30.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:30.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:30.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:13:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.476166 6973 sys_catalog.cc:455] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a6251c96b8534d898aff862b1c9e3140. Latest consensus state: current_term: 1 leader_uuid: "a6251c96b8534d898aff862b1c9e3140" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } } }
20:13:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.477418 6973 sys_catalog.cc:458] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [sys.catalog]: This master's current role is: FOLLOWER
20:13:30.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.481289 6986 sys_catalog.cc:455] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a6251c96b8534d898aff862b1c9e3140" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } } }
20:13:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.483001 6986 sys_catalog.cc:458] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14 [sys.catalog]: This master's current role is: FOLLOWER
20:13:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.487215 6962 sys_catalog.cc:455] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a6251c96b8534d898aff862b1c9e3140" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "92ea8f927bc44b4281ae8b0dbe29df14" member_type: VOTER last_known_addr { host: "127.6.116.254" port: 40471 } } peers { permanent_uuid: "a6251c96b8534d898aff862b1c9e3140" member_type: VOTER last_known_addr { host: "127.6.116.253" port: 38251 } } peers { permanent_uuid: "2105a753f124483f9bef5878e5cf8777" member_type: VOTER last_known_addr { host: "127.6.116.252" port: 42795 } } }
20:13:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.488217 6962 sys_catalog.cc:458] T 00000000000000000000000000000000 P a6251c96b8534d898aff862b1c9e3140 [sys.catalog]: This master's current role is: LEADER
20:13:30.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.553948 6997 catalog_manager.cc:1259] Loaded cluster ID: 5dfb40bc5595400ca65c3af3b51a1411
20:13:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.560343 6997 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14: loading cluster ID for follower catalog manager: success
20:13:30.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.593715 6997 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14: acquiring CA information for follower catalog manager: success
20:13:30.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.601174 6997 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 92ea8f927bc44b4281ae8b0dbe29df14: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:13:30.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.752393 6939 catalog_manager.cc:1259] Loaded cluster ID: 5dfb40bc5595400ca65c3af3b51a1411
20:13:30.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.753258 6939 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777: loading cluster ID for follower catalog manager: success
20:13:30.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.775444 6939 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777: acquiring CA information for follower catalog manager: success
20:13:30.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:30.784552 6939 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2105a753f124483f9bef5878e5cf8777: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:13:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:31.158136 6995 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:31.159255 6995 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:31.160233 6995 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:31.227259 6995 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:31.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:31.228998 6995 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.116.193
20:13:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:31.304857 6995 tablet_server_runner.cc:78] Tablet server non-default flags:
20:13:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/data
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/wal
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.193:0
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/data/info.pb
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.193
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.116.193
20:13:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/logs
20:13:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:13:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:31.307082 6995 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:31.309988 6995 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:31.366999 7007 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:31.380576 7010 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:31.380973 6995 server_base.cc:1034] running on GCE node
20:13:33.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:31.374240 7006 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:33.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.081741 6995 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:33.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.088683 6995 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.090327 6995 hybrid_clock.cc:648] HybridClock initialized: now 1786133613090231 us; error 125 us; skew 500 ppm
20:13:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.091922 6995 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:33.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.109218 6995 webserver.cc:460] Webserver started at http://127.6.116.193:40179/ using document root <none> and password file <none>
20:13:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.110780 6995 fs_manager.cc:359] Metadata directory not provided
20:13:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.111111 6995 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.111783 6995 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:13:33.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.120257 6995 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/data/instance:
20:13:33.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58e69dcc53c04ff5bfda9be99c06f9aa"
20:13:33.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:33 on dist-test-slave-kvfs"
20:13:33.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.122063 6995 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/wal/instance:
20:13:33.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58e69dcc53c04ff5bfda9be99c06f9aa"
20:13:33.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:33 on dist-test-slave-kvfs"
20:13:33.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.134389 6995 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.012s sys 0.000s
20:13:33.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.144086 7020 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:33.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.146993 6995 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.007s sys 0.000s
20:13:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.147544 6995 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/wal
20:13:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58e69dcc53c04ff5bfda9be99c06f9aa"
20:13:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:33 on dist-test-slave-kvfs"
20:13:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.148113 6995 fs_report.cc:383] FS layout report
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/wal
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/wal
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/data/data
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:33.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.212013 6995 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.214565 6995 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.215270 6995 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:33.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.220067 6995 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:13:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.227752 6995 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:13:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.228174 6995 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
20:13:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.228538 6995 ts_tablet_manager.cc:610] Registered 0 tablets
20:13:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.228766 6995 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:13:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.757550 6995 rpc_server.cc:307] RPC server started. Bound to: 127.6.116.193:46845
20:13:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.757954 7187 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.116.193:46845 every 8 connection(s)
20:13:33.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.762486 6995 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/data/info.pb
20:13:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.768842 6611 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 6995
20:13:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.770157 6611 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-0/wal/instance
20:13:33.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.791785 6611 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:33.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:33.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/wal
20:13:33.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/data
20:13:33.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/logs
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/data/info.pb
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.194:0
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.116.194
20:13:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.194
20:13:33.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:33.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:33.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:33.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:33.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:33.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:13:33.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.862277 7191 heartbeater.cc:347] Connected to a master server at 127.6.116.253:38251
20:13:33.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.862974 7191 heartbeater.cc:464] Registering TS with master...
20:13:33.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.864684 7191 heartbeater.cc:511] Master 127.6.116.253:38251 requested a full tablet report, sending...
20:13:33.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.865320 7196 heartbeater.cc:347] Connected to a master server at 127.6.116.254:40471
20:13:33.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.865725 7196 heartbeater.cc:464] Registering TS with master...
20:13:33.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.866637 7196 heartbeater.cc:511] Master 127.6.116.254:40471 requested a full tablet report, sending...
20:13:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.869624 6769 ts_manager.cc:194] Registered new tserver with Master: 58e69dcc53c04ff5bfda9be99c06f9aa (127.6.116.193:46845)
20:13:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.871353 6670 ts_manager.cc:194] Registered new tserver with Master: 58e69dcc53c04ff5bfda9be99c06f9aa (127.6.116.193:46845)
20:13:33.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.873976 6769 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.116.193:52495
20:13:33.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.881467 7192 heartbeater.cc:347] Connected to a master server at 127.6.116.252:42795
20:13:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.881896 7192 heartbeater.cc:464] Registering TS with master...
20:13:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.882732 7192 heartbeater.cc:511] Master 127.6.116.252:42795 requested a full tablet report, sending...
20:13:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:33.886332 6865 ts_manager.cc:194] Registered new tserver with Master: 58e69dcc53c04ff5bfda9be99c06f9aa (127.6.116.193:46845)
20:13:34.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:34.279017 7199 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:34.280006 7199 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:34.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:34.280942 7199 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:34.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:34.320489 7199 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:34.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:34.321661 7199 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.116.194
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:34.364373 7199 tablet_server_runner.cc:78] Tablet server non-default flags:
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/data
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/wal
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.194:0
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/data/info.pb
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.194
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.116.194
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/logs
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:34.366431 7199 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:34.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:34.369277 7199 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:34.403249 7209 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:34.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:34.881496 7191 heartbeater.cc:503] Master 127.6.116.253:38251 was elected leader, sending a full tablet report...
20:13:36.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:34.403345 7210 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:36.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:35.794246 7208 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 7199
20:13:36.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:36.397442 7213 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1994 milliseconds
20:13:36.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:36.395869 7199 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.997s user 0.620s sys 1.274s
20:13:36.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:36.398012 7199 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.999s user 0.620s sys 1.275s
20:13:36.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.398809 7199 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:13:36.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:36.398938 7214 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:36.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.403497 7199 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:36.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.406390 7199 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:36.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.407779 7199 hybrid_clock.cc:648] HybridClock initialized: now 1786133616407733 us; error 51 us; skew 500 ppm
20:13:36.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.408810 7199 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.417155 7199 webserver.cc:460] Webserver started at http://127.6.116.194:43859/ using document root <none> and password file <none>
20:13:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.418277 7199 fs_manager.cc:359] Metadata directory not provided
20:13:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.418511 7199 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.419001 7199 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:13:36.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.425446 7199 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/data/instance:
20:13:36.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30dcb8a71ea043fb914ca9aae6340024"
20:13:36.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:36 on dist-test-slave-kvfs"
20:13:36.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.426937 7199 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/wal/instance:
20:13:36.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30dcb8a71ea043fb914ca9aae6340024"
20:13:36.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:36 on dist-test-slave-kvfs"
20:13:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.438241 7199 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.008s sys 0.004s
20:13:36.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.447616 7225 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:36.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.449615 7199 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.004s sys 0.003s
20:13:36.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.450106 7199 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/wal
20:13:36.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30dcb8a71ea043fb914ca9aae6340024"
20:13:36.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:36 on dist-test-slave-kvfs"
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.450667 7199 fs_report.cc:383] FS layout report
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/wal
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/wal
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/data/data
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:36.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:36.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.517537 7199 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.519461 7199 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.519990 7199 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:36.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.524142 7199 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:13:36.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.530736 7199 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:13:36.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.531025 7199 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:13:36.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.531399 7199 ts_tablet_manager.cc:610] Registered 0 tablets
20:13:36.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:36.531629 7199 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:13:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.035400 7199 rpc_server.cc:307] RPC server started. Bound to: 127.6.116.194:43419
20:13:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.036127 7383 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.116.194:43419 every 8 connection(s)
20:13:37.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.039063 7199 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/data/info.pb
20:13:37.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.051582 6611 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 7199
20:13:37.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.052188 6611 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-1/wal/instance
20:13:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.094491 6611 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/wal
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/data
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/logs
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/data/info.pb
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:13:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.195:0
20:13:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.116.195
20:13:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.195
20:13:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:13:37.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.170286 7384 heartbeater.cc:347] Connected to a master server at 127.6.116.253:38251
20:13:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.171352 7384 heartbeater.cc:464] Registering TS with master...
20:13:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.173394 7384 heartbeater.cc:511] Master 127.6.116.253:38251 requested a full tablet report, sending...
20:13:37.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.175130 7385 heartbeater.cc:347] Connected to a master server at 127.6.116.252:42795
20:13:37.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.175880 7385 heartbeater.cc:464] Registering TS with master...
20:13:37.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.176302 7391 heartbeater.cc:347] Connected to a master server at 127.6.116.254:40471
20:13:37.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.176812 7391 heartbeater.cc:464] Registering TS with master...
20:13:37.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.177809 7385 heartbeater.cc:511] Master 127.6.116.252:42795 requested a full tablet report, sending...
20:13:37.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.178686 7391 heartbeater.cc:511] Master 127.6.116.254:40471 requested a full tablet report, sending...
20:13:37.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.179769 6769 ts_manager.cc:194] Registered new tserver with Master: 30dcb8a71ea043fb914ca9aae6340024 (127.6.116.194:43419)
20:13:37.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.183269 6865 ts_manager.cc:194] Registered new tserver with Master: 30dcb8a71ea043fb914ca9aae6340024 (127.6.116.194:43419)
20:13:37.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.183889 6769 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.116.194:33487
20:13:37.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.185460 6670 ts_manager.cc:194] Registered new tserver with Master: 30dcb8a71ea043fb914ca9aae6340024 (127.6.116.194:43419)
20:13:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:37.784839 7398 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:37.785666 7398 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:37.786185 7398 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:37.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:37.829784 7398 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:37.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:37.830757 7398 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.116.195
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.871804 7398 tablet_server_runner.cc:78] Tablet server non-default flags:
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.116.212:35493
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/data
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/wal
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.116.195:0
20:13:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/data/info.pb
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.116.195
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.116.254:40471,127.6.116.253:38251,127.6.116.252:42795
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.116.195
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/logs
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.873924 7398 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:37.876299 7398 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:37.929756 7407 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:38.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:38.192695 7384 heartbeater.cc:503] Master 127.6.116.253:38251 was elected leader, sending a full tablet report...
20:13:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:37.931190 7409 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:39.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:39.280839 7412 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:39.283505 7410 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1352 milliseconds
20:13:39.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.283633 7398 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:13:39.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.285158 7398 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:39.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.288568 7398 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:39.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.290096 7398 hybrid_clock.cc:648] HybridClock initialized: now 1786133619290058 us; error 59 us; skew 500 ppm
20:13:39.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.291064 7398 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:39.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.303136 7398 webserver.cc:460] Webserver started at http://127.6.116.195:42201/ using document root <none> and password file <none>
20:13:39.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.304399 7398 fs_manager.cc:359] Metadata directory not provided
20:13:39.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.304634 7398 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:39.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.305140 7398 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:13:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.311892 7398 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/data/instance:
20:13:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0996ace0ffb2400a808257b4461b520c"
20:13:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:39 on dist-test-slave-kvfs"
20:13:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.313519 7398 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/wal/instance:
20:13:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0996ace0ffb2400a808257b4461b520c"
20:13:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:39 on dist-test-slave-kvfs"
20:13:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.322912 7398 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
20:13:39.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.329865 7424 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:39.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.331236 7398 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
20:13:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.331681 7398 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/wal
20:13:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0996ace0ffb2400a808257b4461b520c"
20:13:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:39 on dist-test-slave-kvfs"
20:13:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.332139 7398 fs_report.cc:383] FS layout report
20:13:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/wal
20:13:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/wal
20:13:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/data/data
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.398079 7398 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.401798 7398 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.402725 7398 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:39.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.411199 7398 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:13:39.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.421178 7398 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:13:39.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.421500 7398 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:13:39.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.421893 7398 ts_tablet_manager.cc:610] Registered 0 tablets
20:13:39.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:39.422150 7398 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:13:40.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.025826 7398 rpc_server.cc:307] RPC server started. Bound to: 127.6.116.195:44833
20:13:40.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.026358 7586 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.116.195:44833 every 8 connection(s)
20:13:40.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.030406 7398 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/data/info.pb
20:13:40.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.032908 6611 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 7398
20:13:40.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.033499 6611 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster5641386263261114709/ts-2/wal/instance
20:13:40.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.140858 7597 heartbeater.cc:347] Connected to a master server at 127.6.116.254:40471
20:13:40.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.142022 7597 heartbeater.cc:464] Registering TS with master...
20:13:40.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.142007 7588 heartbeater.cc:347] Connected to a master server at 127.6.116.253:38251
20:13:40.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.142707 7588 heartbeater.cc:464] Registering TS with master...
20:13:40.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.144595 7597 heartbeater.cc:511] Master 127.6.116.254:40471 requested a full tablet report, sending...
20:13:40.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.144708 7588 heartbeater.cc:511] Master 127.6.116.253:38251 requested a full tablet report, sending...
20:13:40.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.149559 6769 ts_manager.cc:194] Registered new tserver with Master: 0996ace0ffb2400a808257b4461b520c (127.6.116.195:44833)
20:13:40.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.150386 6670 ts_manager.cc:194] Registered new tserver with Master: 0996ace0ffb2400a808257b4461b520c (127.6.116.195:44833)
20:13:40.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.150744 7589 heartbeater.cc:347] Connected to a master server at 127.6.116.252:42795
20:13:40.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.151224 7589 heartbeater.cc:464] Registering TS with master...
20:13:40.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.153074 7589 heartbeater.cc:511] Master 127.6.116.252:42795 requested a full tablet report, sending...
20:13:40.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.153266 6769 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.116.195:43611
20:13:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.159281 6865 ts_manager.cc:194] Registered new tserver with Master: 0996ace0ffb2400a808257b4461b520c (127.6.116.195:44833)
20:13:40.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:40.179780 6611 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:13:40.206 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:13:40.208 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:13:40.222 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.116.254"
port: 40471
}
bound_http_address {
host: "127.6.116.254"
port: 33013
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.116.253"
port: 38251
}
bound_http_address {
host: "127.6.116.253"
port: 36151
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.116.252"
port: 42795
}
bound_http_address {
host: "127.6.116.252"
port: 43097
}
}
}
20:13:40.228 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:13:40.232 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.6.116.193"
port: 46845
}
bound_http_address {
host: "127.6.116.193"
port: 40179
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.6.116.194"
port: 43419
}
bound_http_address {
host: "127.6.116.194"
port: 43859
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.6.116.195"
port: 44833
}
bound_http_address {
host: "127.6.116.195"
port: 42201
}
}
}
20:13:40.233 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:13:40.487 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:13:40.680 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasknE2NB6/test-tmp
20:13:40.682 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:13:40.682 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasknE2NB6/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:13:40.707 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:13:41.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:41.161705 7588 heartbeater.cc:503] Master 127.6.116.253:38251 was elected leader, sending a full tablet report...
20:13:41.286 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:41.287 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:13:41.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:41.291050 7607 mini_oidc.cc:152] Starting JWKS server
20:13:41.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:41.298525 7607 webserver.cc:460] Webserver started at http://127.0.0.1:42133/ using document root <none> and password file <none>
20:13:41.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:41.299335 7607 mini_oidc.cc:189] Starting OIDC Discovery server
20:13:41.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:41.302572 7607 webserver.cc:460] Webserver started at http://127.0.0.1:36271/ using document root <none> and password file <none>
20:13:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:13:41.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/principal' for realm 'KRBTEST.COM',
20:13:41.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:13:41.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:41 dist-test-slave-kvfs krb5kdc[7625](info): setting up network...
20:13:41.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:13:41.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:41 dist-test-slave-kvfs krb5kdc[7625](info): set up 2 sockets
20:13:41.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:41 dist-test-slave-kvfs krb5kdc[7625](info): commencing operation
20:13:41.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:13:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:42.819262 7607 mini_kdc.cc:121] Time spent starting KDC: real 1.480s user 0.002s sys 0.035s
20:13:42.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:13:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:13:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:13:42.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:13:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:13:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:13:42.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:13:42.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:13:42.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:13:42.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:13:42.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/test-user.keytab.
20:13:42.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/test-user.keytab.
20:13:43.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:43 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133623, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:13:43.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:13:43.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T20:13:43Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:13:43.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T20:13:43Z Disabled control of system clock
20:13:43.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.109.254@KRBTEST.COM; defaulting to no policy
20:13:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:13:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.109.254@KRBTEST.COM" created.
20:13:43.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:13:43.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.109.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.254.keytab.
20:13:43.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.109.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.254.keytab.
20:13:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:43.134485 7607 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal
20:13:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data
20:13:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/logs
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data/info.pb
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.254:39805
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.254
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.254.keytab
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.254
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:43.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:13:43.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.768718 7643 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:43.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.769840 7643 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:43.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.770710 7643 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.809293 7643 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:13:43.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.809667 7643 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:13:43.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.810148 7643 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:43.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.810551 7643 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:13:43.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.810909 7643 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:13:43.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.811259 7643 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:13:43.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.811525 7643 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42133/jwks.json
20:13:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:43.859086 7643 master_runner.cc:386] Master server non-default flags:
20:13:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data
20:13:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal
20:13:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:13:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.254.keytab
20:13:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.254
20:13:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.254:39805
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data/info.pb
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.254
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:43.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/logs
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:43.861104 7643 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:43.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:43.862777 7643 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:43.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.889734 7655 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:45.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.889783 7658 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:45.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:43.892861 7643 server_base.cc:1034] running on GCE node
20:13:45.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:43.889935 7656 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:45.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.275720 7643 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:45.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.281890 7643 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:45.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.283672 7643 hybrid_clock.cc:648] HybridClock initialized: now 1786133625283594 us; error 123 us; skew 500 ppm
20:13:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.285005 7643 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:45.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:45 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133625, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:13:45.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.305540 7643 init.cc:377] Logged in from keytab as kudu/127.7.109.254@KRBTEST.COM (short username kudu)
20:13:45.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.318988 7643 webserver.cc:460] Webserver started at http://127.7.109.254:44923/ using document root <none> and password file <none>
20:13:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.320273 7643 fs_manager.cc:359] Metadata directory not provided
20:13:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.320536 7643 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:45.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.321209 7643 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:13:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.328670 7643 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data/instance:
20:13:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbcc3cd8263042c79af252ac96efca5d"
20:13:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:45 on dist-test-slave-kvfs"
20:13:45.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.330283 7643 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal/instance:
20:13:45.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbcc3cd8263042c79af252ac96efca5d"
20:13:45.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:45 on dist-test-slave-kvfs"
20:13:45.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.344834 7643 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.009s sys 0.005s
20:13:45.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.357424 7674 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:45.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.359241 7643 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.004s sys 0.004s
20:13:45.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.359629 7643 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbcc3cd8263042c79af252ac96efca5d"
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:45 on dist-test-slave-kvfs"
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.360139 7643 fs_report.cc:383] FS layout report
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data/data
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:45.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.423202 7643 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:45.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.433478 7643 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:45.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.434520 7643 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:45.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.775610 7643 rpc_server.cc:307] RPC server started. Bound to: 127.7.109.254:39805
20:13:45.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.775825 7739 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.109.254:39805 every 8 connection(s)
20:13:45.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.780757 7643 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data/info.pb
20:13:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.791414 7607 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 7643
20:13:45.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.793785 7607 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal/instance
20:13:45.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.798656 7740 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:13:45.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.826239 7740 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } has no permanent_uuid. Determining permanent_uuid...
20:13:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.109.253@KRBTEST.COM; defaulting to no policy
20:13:45.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:13:45.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.109.253@KRBTEST.COM" created.
20:13:45.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:45 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133625, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.254@KRBTEST.COM for kudu/127.7.109.254@KRBTEST.COM
20:13:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:13:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.109.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.253.keytab.
20:13:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.109.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.253.keytab.
20:13:45.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.895062 7607 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:45.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:45.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal
20:13:45.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data
20:13:45.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:45.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:45.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:45.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/logs
20:13:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data/info.pb
20:13:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.253:35947
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.253
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.253.keytab
20:13:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.253
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:13:45.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.902169 7740 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } has no permanent_uuid. Determining permanent_uuid...
20:13:45.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:45.907227 7678 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:45.917215 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.253:35947: Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:45.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:45.967248 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } attempt: 1
20:13:45.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:45.973596 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.253:35947: Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:46.042112 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } attempt: 2
20:13:46.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.046505 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.253:35947: Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:46.138367 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } attempt: 3
20:13:46.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.144497 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.253:35947: Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:46.288214 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } attempt: 4
20:13:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.295518 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.253:35947: Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:46.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.570511 7750 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:46.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.571971 7750 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:46.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.572764 7750 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:46.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:46.595436 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } attempt: 5
20:13:46.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.601280 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.253:35947: Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.616401 7750 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:13:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.616758 7750 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:13:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.617165 7750 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.617406 7750 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:13:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.617610 7750 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:13:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.617802 7750 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:13:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.617957 7750 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42133/jwks.json
20:13:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:46.663228 7750 master_runner.cc:386] Master server non-default flags:
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.253.keytab
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.253
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.253:35947
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data/info.pb
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.253
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/logs
20:13:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:13:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:46.664737 7750 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:46.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:46.666349 7750 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.713263 7762 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:47.149151 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } attempt: 6
20:13:47.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:47.174153 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.253:35947: Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:48.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.235321 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } attempt: 7
20:13:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:48.267946 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.253:35947: Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:48.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.713593 7759 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:48.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:46.713990 7758 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:48.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:48.377275 7761 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1666 milliseconds
20:13:48.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.377431 7750 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:13:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.379199 7750 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:48.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.384393 7750 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:48.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.385859 7750 hybrid_clock.cc:648] HybridClock initialized: now 1786133628385816 us; error 57 us; skew 500 ppm
20:13:48.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.387157 7750 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:48 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133628, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:13:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.400405 7750 init.cc:377] Logged in from keytab as kudu/127.7.109.253@KRBTEST.COM (short username kudu)
20:13:48.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.417030 7750 webserver.cc:460] Webserver started at http://127.7.109.253:39551/ using document root <none> and password file <none>
20:13:48.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.418540 7750 fs_manager.cc:359] Metadata directory not provided
20:13:48.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.418825 7750 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:48.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.419445 7750 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:13:48.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.426767 7750 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data/instance:
20:13:48.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "122b53cc6dab4daa8dc015f29f856547"
20:13:48.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:48 on dist-test-slave-kvfs"
20:13:48.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.428552 7750 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal/instance:
20:13:48.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "122b53cc6dab4daa8dc015f29f856547"
20:13:48.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:48 on dist-test-slave-kvfs"
20:13:48.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.444346 7750 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.013s sys 0.000s
20:13:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.457559 7781 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:48.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.459373 7750 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.006s sys 0.003s
20:13:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.459779 7750 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal
20:13:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "122b53cc6dab4daa8dc015f29f856547"
20:13:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:48 on dist-test-slave-kvfs"
20:13:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.460263 7750 fs_report.cc:383] FS layout report
20:13:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data/data
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:48.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.527818 7750 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:48.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.542256 7750 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:48.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.543447 7750 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:48.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.869419 7750 rpc_server.cc:307] RPC server started. Bound to: 127.7.109.253:35947
20:13:48.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.869678 7855 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.109.253:35947 every 8 connection(s)
20:13:48.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.873039 7750 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data/info.pb
20:13:48.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.878770 7607 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 7750
20:13:48.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.879490 7607 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal/instance
20:13:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.891866 7856 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:13:48.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.109.252@KRBTEST.COM; defaulting to no policy
20:13:48.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.923923 7856 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } has no permanent_uuid. Determining permanent_uuid...
20:13:48.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:13:48.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.109.252@KRBTEST.COM" created.
20:13:48.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:48 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133628, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.253@KRBTEST.COM for kudu/127.7.109.254@KRBTEST.COM
20:13:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:13:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.109.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.252.keytab.
20:13:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.109.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.252.keytab.
20:13:48.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.982038 7607 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:48.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:48.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal
20:13:48.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data
20:13:48.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:48.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:48.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/logs
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data/info.pb
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:48.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.252:36293
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.252
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.252.keytab
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.252
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:48.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:13:48.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:48.995260 7856 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } has no permanent_uuid. Determining permanent_uuid...
20:13:49.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:49 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133628, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.253@KRBTEST.COM for kudu/127.7.109.253@KRBTEST.COM
20:13:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:49.050292 7856 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } has no permanent_uuid. Determining permanent_uuid...
20:13:49.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.055073 7784 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:49.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.067690 7856 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:49.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:49.117708 7856 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 1
20:13:49.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.124176 7856 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:49.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:49.192900 7856 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 2
20:13:49.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.199983 7856 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:49.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:49.291541 7856 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 3
20:13:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.300010 7856 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:49.444015 7856 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 4
20:13:49.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.452814 7856 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.672042 7863 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.673051 7863 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.673875 7863 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.711279 7863 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:13:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.711691 7863 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:13:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.712234 7863 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.712625 7863 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:13:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.712973 7863 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:13:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.713310 7863 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:13:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.713557 7863 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42133/jwks.json
20:13:49.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:49.752586 7856 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 5
20:13:49.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.759733 7856 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:49.764132 7863 master_runner.cc:386] Master server non-default flags:
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.252.keytab
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.252
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.252:36293
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data/info.pb
20:13:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.252
20:13:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:13:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/logs
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:49.766525 7863 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:49.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:49.768182 7863 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.811924 7874 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:50.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:50.307370 7856 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 6
20:13:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:50.332093 7856 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:50.358991 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } attempt: 8
20:13:50.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:50 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133625, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.254@KRBTEST.COM for kudu/127.7.109.253@KRBTEST.COM
20:13:50.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:50.452353 7740 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } has no permanent_uuid. Determining permanent_uuid...
20:13:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:50.482059 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:50.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:50.548213 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 1
20:13:50.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:50.567420 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:50.621582 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 2
20:13:50.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:50.630215 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:50.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:50.706962 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 3
20:13:50.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:50.717908 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:50.873910 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 4
20:13:50.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:50.887253 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.184093 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 5
20:13:51.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:51.188251 7678 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111) [suppressed 12 similar messages]
20:13:51.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:51.193552 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:51.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.393087 7856 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 7
20:13:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:51.418610 7856 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:51.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.818872 7876 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:51.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:49.812426 7873 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:51.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:51.500598 7875 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1688 milliseconds
20:13:51.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.500777 7863 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:13:51.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.503770 7863 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:51.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.509011 7863 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:51.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.510710 7863 hybrid_clock.cc:648] HybridClock initialized: now 1786133631510679 us; error 50 us; skew 500 ppm
20:13:51.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.511641 7863 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:51 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133631, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:13:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.526747 7863 init.cc:377] Logged in from keytab as kudu/127.7.109.252@KRBTEST.COM (short username kudu)
20:13:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.545197 7863 webserver.cc:460] Webserver started at http://127.7.109.252:44885/ using document root <none> and password file <none>
20:13:51.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.546929 7863 fs_manager.cc:359] Metadata directory not provided
20:13:51.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.547257 7863 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:51.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.547928 7863 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:13:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.557910 7863 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data/instance:
20:13:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51614eb31644f6f91fa39a6c96fcaf0"
20:13:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:51 on dist-test-slave-kvfs"
20:13:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.559660 7863 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal/instance:
20:13:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51614eb31644f6f91fa39a6c96fcaf0"
20:13:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:51 on dist-test-slave-kvfs"
20:13:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.574142 7863 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.006s sys 0.006s
20:13:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.586189 7899 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.588240 7863 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.002s sys 0.008s
20:13:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.588788 7863 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal
20:13:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51614eb31644f6f91fa39a6c96fcaf0"
20:13:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:51 on dist-test-slave-kvfs"
20:13:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.589311 7863 fs_report.cc:383] FS layout report
20:13:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data/data
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.667748 7863 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.676545 7863 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:51.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.677389 7863 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:51.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.715368 7740 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } attempt: 6
20:13:51.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:51.730021 7740 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.109.252:36293: Network error: Client connection negotiation failed: client connection to 127.7.109.252:36293: connect: Connection refused (error 111)
20:13:51.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.994400 7863 rpc_server.cc:307] RPC server started. Bound to: 127.7.109.252:36293
20:13:51.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:51.994532 7972 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.109.252:36293 every 8 connection(s)
20:13:52.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.001803 7863 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data/info.pb
20:13:52.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.013288 7607 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 7863
20:13:52.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.016079 7607 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal/instance
20:13:52.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.025305 7974 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:13:52.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.087448 7974 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } has no permanent_uuid. Determining permanent_uuid...
20:13:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:52 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133623, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.109.252@KRBTEST.COM
20:13:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:52 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133631, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.252@KRBTEST.COM for kudu/127.7.109.254@KRBTEST.COM
20:13:52.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.167259 7974 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } has no permanent_uuid. Determining permanent_uuid...
20:13:52.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:52 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133623, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.109.253@KRBTEST.COM
20:13:52.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:52 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133631, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.252@KRBTEST.COM for kudu/127.7.109.253@KRBTEST.COM
20:13:52.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:52 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133623, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.109.254@KRBTEST.COM
20:13:52.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.233423 7974 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } has no permanent_uuid. Determining permanent_uuid...
20:13:52.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:52 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133631, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.252@KRBTEST.COM for kudu/127.7.109.252@KRBTEST.COM
20:13:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.272653 7607 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
20:13:52.302 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.299299 7974 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: Bootstrap starting.
20:13:52.304 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:13:52.322 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:13:52.323 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:13:52.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.327621 7974 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: Neither blocks nor log segments found. Creating new log.
20:13:52.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.334679 7974 log.cc:824] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: Log is configured to *not* fsync() on all Append() calls
20:13:52.335 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.109.254"
port: 39805
}
bound_http_address {
host: "127.7.109.254"
port: 44923
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.109.253"
port: 35947
}
bound_http_address {
host: "127.7.109.253"
port: 39551
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.109.252"
port: 36293
}
bound_http_address {
host: "127.7.109.252"
port: 44885
}
}
}
20:13:52.336 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:13:52.337 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:52.337 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:13:52.339 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:13:52.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.349228 7974 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: No bootstrap required, opened a new log
20:13:52.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:52 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133632, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:13:52.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:13:52.397 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:52.400 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5cc
20:13:52.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.407536 7974 raft_consensus.cc:348] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:13:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.408869 7974 raft_consensus.cc:374] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:13:52.409 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:13:52.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.409379 7974 raft_consensus.cc:729] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f51614eb31644f6f91fa39a6c96fcaf0, State: Initialized, Role: FOLLOWER
20:13:52.411 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5cc
20:13:52.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.411079 7974 consensus_queue.cc:260] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:13:52.413 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:13:52.420 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:13:52.423 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:13:52.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.421443 7990 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } } }
20:13:52.424 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:13:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.423620 7990 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [sys.catalog]: This master's current role is: FOLLOWER
20:13:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.424718 7974 sys_catalog.cc:564] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [sys.catalog]: configured and running, proceeding with master startup.
20:13:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:52.444458 7908 tablet.cc:2369] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:13:52.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:52 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133632, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:13:52.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:13:52.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:52.510 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5cc
20:13:52.512 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:13:52.513 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.109.254:39805
20:13:52.519 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:13:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.520646 7607 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu with pid 7643
20:13:52.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:52.582651 8008 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:13:52.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:52.583055 8008 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:13:52.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.520009 7863 master_runner.cc:186] Error getting master registration for 127.7.109.254:39805: OK, instance_id { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" instance_seqno: 1786133625678065 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:13:52.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.591645 7863 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:13:52.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:52.595 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.109.252:36293
20:13:52.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:13:52.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.595954 7607 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu with pid 7863
20:13:52.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:52.626 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.109.253:35947
20:13:52.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:13:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.627089 7607 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu with pid 7750
20:13:52.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:52.665 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.109.254:39805
20:13:52.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:13:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:52.669059 7607 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal
20:13:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data
20:13:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/logs
20:13:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data/info.pb
20:13:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.254:39805
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.254
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44923
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.254.keytab
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.254
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:52.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:52.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:52.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:52.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:52.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:52.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:13:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.055531 8010 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.056480 8010 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:53.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.057493 8010 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:53.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.095098 8010 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:13:53.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.095469 8010 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:13:53.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.095945 8010 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:53.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.096397 8010 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:13:53.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.096747 8010 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:13:53.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.096976 8010 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:13:53.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.097126 8010 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42133/jwks.json
20:13:53.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:53.141330 8010 master_runner.cc:386] Master server non-default flags:
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.254.keytab
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.254
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.254:39805
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:53.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:53.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data/info.pb
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.254
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44923
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/logs
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:53.142908 8010 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:53.144220 8010 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.167133 8020 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.167222 8019 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:53.169018 8022 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:54.468182 8021 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:13:54.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.468802 8010 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:13:54.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.473918 8010 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.478030 8010 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.479573 8010 hybrid_clock.cc:648] HybridClock initialized: now 1786133634479528 us; error 75 us; skew 500 ppm
20:13:54.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.480922 8010 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:54.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:54 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133634, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:13:54.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.489114 8010 init.cc:377] Logged in from keytab as kudu/127.7.109.254@KRBTEST.COM (short username kudu)
20:13:54.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.495620 8010 webserver.cc:460] Webserver started at http://127.7.109.254:44923/ using document root <none> and password file <none>
20:13:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.496824 8010 fs_manager.cc:359] Metadata directory not provided
20:13:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.497208 8010 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:54.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.508265 8010 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.002s sys 0.005s
20:13:54.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.513733 8037 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:54.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.515230 8010 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
20:13:54.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.515585 8010 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal
20:13:54.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbcc3cd8263042c79af252ac96efca5d"
20:13:54.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:45 on dist-test-slave-kvfs"
20:13:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.518491 8010 fs_report.cc:383] FS layout report
20:13:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal
20:13:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal
20:13:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data/data
20:13:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:54.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.595801 8010 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.603915 8010 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.604574 8010 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.698734 8010 rpc_server.cc:307] RPC server started. Bound to: 127.7.109.254:39805
20:13:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.698817 8111 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.109.254:39805 every 8 connection(s)
20:13:54.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.702431 8010 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/data/info.pb
20:13:54.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.707399 7607 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 8010
20:13:54.708 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:54.708 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.109.252:36293
20:13:54.708 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.710678 7607 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/logs
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data/info.pb
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.252:36293
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.252
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44885
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.252.keytab
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.252
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:54.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:54.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:54.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:13:54.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.720589 8112 sys_catalog.cc:263] Verifying existing consensus state
20:13:54.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260807 20:13:54.722306 8112 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:13:54.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.723034 8010 master.cc:537] Master@127.7.109.254:39805 shutting down...
20:13:54.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:54.806350 8010 master.cc:552] Master@127.7.109.254:39805 shutdown complete.
20:13:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:13:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.134620 8114 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.135353 8114 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.136364 8114 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:55.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.172500 8114 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:13:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.172789 8114 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:13:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.173120 8114 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.173342 8114 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:13:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.173530 8114 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:13:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.173727 8114 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:13:55.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.173964 8114 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42133/jwks.json
20:13:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:55.217782 8114 master_runner.cc:386] Master server non-default flags:
20:13:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data
20:13:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal
20:13:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:13:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.252.keytab
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.252
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.252:36293
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data/info.pb
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.252
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44885
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/logs
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:13:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:55.219450 8114 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:55.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:55.220820 8114 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.244102 8126 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:56.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.251789 8124 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:56.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:55.244086 8121 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:56.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:56.684146 8125 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1435 milliseconds
20:13:56.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.684322 8114 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:13:56.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.685851 8114 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:56.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.689724 8114 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.691243 8114 hybrid_clock.cc:648] HybridClock initialized: now 1786133636691184 us; error 62 us; skew 500 ppm
20:13:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.692334 8114 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:56.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:56 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133636, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:13:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.704664 8114 init.cc:377] Logged in from keytab as kudu/127.7.109.252@KRBTEST.COM (short username kudu)
20:13:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.711755 8114 webserver.cc:460] Webserver started at http://127.7.109.252:44885/ using document root <none> and password file <none>
20:13:56.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.713147 8114 fs_manager.cc:359] Metadata directory not provided
20:13:56.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.713430 8114 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:56.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.723960 8114 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.003s sys 0.004s
20:13:56.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.730086 8142 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:56.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.731657 8114 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.732131 8114 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51614eb31644f6f91fa39a6c96fcaf0"
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:51 on dist-test-slave-kvfs"
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.735044 8114 fs_report.cc:383] FS layout report
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/wal
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data/data
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:56.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:56.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:56.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:56.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:56.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:56.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:56.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:56.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:56.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.786621 8114 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:56.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.794252 8114 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:56.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.794857 8114 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:56.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.892417 8114 rpc_server.cc:307] RPC server started. Bound to: 127.7.109.252:36293
20:13:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.892591 8215 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.109.252:36293 every 8 connection(s)
20:13:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.896102 8114 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-2/data/info.pb
20:13:56.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.905001 7607 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 8114
20:13:56.906 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:56.906 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.109.253:35947
20:13:56.907 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:13:56.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.909297 7607 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:56.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu
20:13:56.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal
20:13:56.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/logs
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data/info.pb
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.253:35947
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.253
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39551
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.253.keytab
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.253
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:13:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:56.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:56.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:56.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:56.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:56.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:56.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:56.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:56.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:13:56.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.911864 8216 sys_catalog.cc:263] Verifying existing consensus state
20:13:56.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.919128 8216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: Bootstrap starting.
20:13:56.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.975200 8216 log.cc:824] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: Log is configured to *not* fsync() on all Append() calls
20:13:56.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.982070 8216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:13:56.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:56.983250 8216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: Bootstrap complete.
20:13:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.017637 8216 raft_consensus.cc:348] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:13:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.020421 8216 raft_consensus.cc:374] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:13:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.020749 8216 raft_consensus.cc:729] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f51614eb31644f6f91fa39a6c96fcaf0, State: Initialized, Role: FOLLOWER
20:13:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.021880 8216 consensus_queue.cc:260] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:13:57.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.038029 8222 sys_catalog.cc:455] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } } }
20:13:57.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.040000 8222 sys_catalog.cc:458] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [sys.catalog]: This master's current role is: FOLLOWER
20:13:57.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.041134 8216 sys_catalog.cc:564] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [sys.catalog]: configured and running, proceeding with master startup.
20:13:57.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.051872 8150 tablet.cc:2369] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:13:57.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.085466 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:13:57.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.085978 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:13:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.090317 8114 master_runner.cc:186] Error getting master registration for 127.7.109.254:39805: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111),
20:13:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.113880 8114 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:13:57.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.375075 8219 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:13:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.375811 8219 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:13:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.376641 8219 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:13:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.415575 8219 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:13:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.415946 8219 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:13:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.416332 8219 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:13:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.416560 8219 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:13:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.416755 8219 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:13:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.416958 8219 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:13:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.417107 8219 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42133/jwks.json
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.465471 8219 master_runner.cc:386] Master server non-default flags:
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.109.212:34535
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.109.254:39805,127.7.109.253:35947,127.7.109.252:36293
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/krb5kdc/kudu_127.7.109.253.keytab
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.109.253
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.109.253:35947
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42133/jwks.json
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:13:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data/info.pb
20:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.109.253
20:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39551
20:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:13:57.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/logs
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 18:36:40 UTC on ccycloud.quasar-chjqtz.root.comops.site
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4573193
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:13:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.467547 8219 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:13:57.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.469221 8219 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:13:57.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.508642 8222 raft_consensus.cc:482] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:13:57.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.509320 8222 raft_consensus.cc:504] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:13:57.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.512739 8222 leader_election.cc:288] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805), 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947)
20:13:57.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.516038 8147 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111)
20:13:57.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.532734 8147 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805): Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111)
20:13:57.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.536325 8146 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947): Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:57.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.537130 8146 leader_election.cc:302] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f51614eb31644f6f91fa39a6c96fcaf0; no voters: 122b53cc6dab4daa8dc015f29f856547, bbcc3cd8263042c79af252ac96efca5d
20:13:57.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.538441 8222 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:13:58.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:58.094029 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:13:58.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:58.094655 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:13:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.496970 8256 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.496995 8255 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:57.498813 8219 server_base.cc:1034] running on GCE node
20:13:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:57.496986 8258 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:13:58.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.901113 8219 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:13:58.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.905800 8219 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:13:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.907848 8219 hybrid_clock.cc:648] HybridClock initialized: now 1786133638907748 us; error 133 us; skew 500 ppm
20:13:58.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.909173 8219 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:13:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:58 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133638, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:13:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.917663 8219 init.cc:377] Logged in from keytab as kudu/127.7.109.253@KRBTEST.COM (short username kudu)
20:13:58.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.924628 8219 webserver.cc:460] Webserver started at http://127.7.109.253:39551/ using document root <none> and password file <none>
20:13:58.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.925959 8219 fs_manager.cc:359] Metadata directory not provided
20:13:58.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.926298 8219 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:13:58.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.938410 8219 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.000s sys 0.008s
20:13:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.945010 8280 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:13:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.946519 8219 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.004s sys 0.000s
20:13:58.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.946902 8219 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data,/tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal
20:13:58.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "122b53cc6dab4daa8dc015f29f856547"
20:13:58.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 20:13:48 on dist-test-slave-kvfs"
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:58.949740 8219 fs_report.cc:383] FS layout report
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data/data
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:13:58.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:13:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.003829 8219 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:13:59.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.011433 8219 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:13:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.011963 8219 kserver.cc:163] Server-wide thread pool size limit: 3276
20:13:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:59.099938 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:13:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:59.100608 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:13:59.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.129382 8219 rpc_server.cc:307] RPC server started. Bound to: 127.7.109.253:35947
20:13:59.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.129438 8357 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.109.253:35947 every 8 connection(s)
20:13:59.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.134464 8219 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/data/info.pb
20:13:59.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.141911 7607 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu as pid 8219
20:13:59.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:13:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.149531 8358 sys_catalog.cc:263] Verifying existing consensus state
20:13:59.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260807 20:13:59.150764 8358 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:13:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.151378 8219 master.cc:537] Master@127.7.109.253:35947 shutting down...
20:13:59.216 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:13:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.246107 8219 master.cc:552] Master@127.7.109.253:35947 shutdown complete.
20:13:59.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasknE2NB6/test-tmp/mini-kudu-cluster6786743198114925225/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:13:59.320 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.109.252:36293(127.7.109.252:36293)] connecting to peer
20:13:59.324 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:13:59.380 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.7.109.252:36293(127.7.109.252:36293)] Successfully connected to peer
20:13:59.383 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.396996 8367 raft_consensus.cc:482] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:13:59.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.397886 8367 raft_consensus.cc:504] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:13:59.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.402 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.404 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.408212 8367 leader_election.cc:288] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805), 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947)
20:13:59.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:59.425729 8147 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805): Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111)
20:13:59.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:13:59.428251 8146 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947): Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:13:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.428781 8146 leader_election.cc:302] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f51614eb31644f6f91fa39a6c96fcaf0; no voters: 122b53cc6dab4daa8dc015f29f856547, bbcc3cd8263042c79af252ac96efca5d
20:13:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:13:59.429720 8367 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:13:59.511 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:13:59.525 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.7.109.252
20:13:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:13:59 dist-test-slave-kvfs krb5kdc[7625](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133632, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.109.252@KRBTEST.COM
20:13:59.793 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x923d443a, L:/127.0.0.1:58014 - R:/127.7.109.252:36293] using SASL/GSSAPI
20:13:59.812 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:13:59.831 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:13:59.836 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.836 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.836 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:13:59.837 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.840 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.845 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:13:59.850 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:13:59.852 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:13:59.854 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.854 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.855 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.855 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.855 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.856 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:13:59.869 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:13:59.871 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:13:59.874 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.875 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.876 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.876 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.876 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.877 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.880 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:13:59.909 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:13:59.912 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:13:59.914 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.915 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.916 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.917 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.917 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.918 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:13:59.930 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:13:59.932 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:13:59.934 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.934 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.935 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.935 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.936 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.936 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:13:59.970 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:13:59.973 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:13:59.976 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.976 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.976 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.976 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.977 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:13:59.977 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:13:59.979 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:00.090 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:00.093 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:00.096 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:00.097 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:00.098 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:00.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:00.101 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:00.103 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:00.103 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:00.104993 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:00.105289 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:00.290 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:00.293 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:00.297 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:00.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:00.303 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:00.303 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:00.303 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:00.304 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:00.306 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:00.730 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:00.732 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:00.734 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:00.735 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:00.735 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:00.736 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:00.736 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:00.737 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:00.739 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:01.108618 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:01.109011 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:01.650 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:01.654 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:01.655 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:01.655 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:01.656 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:01.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:01.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:01.657 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:01.658 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:02.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:02.112287 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:02.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:02.112640 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:02.117064 8114 master_runner.cc:186] Error getting master registration for 127.7.109.254:39805: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111),
20:14:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:02.117430 8114 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:14:02.370 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:02.372 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:02.374 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:02.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:02.375 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:02.377 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:02.377 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:02.377 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:02.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:02.410063 8375 raft_consensus.cc:482] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:14:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:02.410595 8375 raft_consensus.cc:504] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:14:02.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:02.412441 8375 leader_election.cc:288] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805), 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947)
20:14:02.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:02.417140 8147 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805): Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111)
20:14:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:02.417862 8146 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947): Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:14:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:02.418301 8146 leader_election.cc:302] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f51614eb31644f6f91fa39a6c96fcaf0; no voters: 122b53cc6dab4daa8dc015f29f856547, bbcc3cd8263042c79af252ac96efca5d
20:14:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:02.419005 8375 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:14:03.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:03.116441 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:03.116751 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:04.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:04.120249 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:04.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:04.120584 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:05.126891 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:05.127357 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:05.410 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:05.412 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:05.415 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:05.415 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:05.415 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:05.415 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:05.416 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:05.416 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:05.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:05.725016 8388 raft_consensus.cc:482] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:14:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:05.725639 8388 raft_consensus.cc:504] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:14:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:05.727808 8388 leader_election.cc:288] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805), 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947)
20:14:05.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:05.730753 8147 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111) [suppressed 5 similar messages]
20:14:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:05.733994 8146 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947): Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:14:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:05.734716 8147 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805): Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111)
20:14:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:05.735155 8147 leader_election.cc:302] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f51614eb31644f6f91fa39a6c96fcaf0; no voters: 122b53cc6dab4daa8dc015f29f856547, bbcc3cd8263042c79af252ac96efca5d
20:14:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:05.736212 8388 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:14:06.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:06.130822 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:06.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:06.131141 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:06.190 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:06.192 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:06.195 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:06.195 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:06.195 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:06.195 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:06.195 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:06.195 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:06.197 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:14:06 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133646, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:14:06.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:06.709712 8134 init.cc:295] Successfully reacquired a new kerberos TGT
20:14:07.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:07.121013 8114 master_runner.cc:186] Error getting master registration for 127.7.109.254:39805: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111),
20:14:07.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:07.121508 8114 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:14:07.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:07.134461 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:07.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:07.134871 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:08.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:08.138432 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:08.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:08.138741 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:09.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:09.142181 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:09.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:09.142496 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:10.050 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:10.051 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:10.053 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:10.054 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:10.054 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:10.055 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:10.056 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:10.056 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:10.056 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:10.145946 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:10.146311 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:10.550 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:10.553 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:10.555 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:10.555 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:10.556 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:10.556 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:10.556 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:10.556 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:10.558 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:11.149755 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:11.150103 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:12.125161 8114 master_runner.cc:186] Error getting master registration for 127.7.109.254:39805: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111),
20:14:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:12.125591 8114 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:14:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:12.153808 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:12.154167 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:12.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:12.917359 8407 raft_consensus.cc:482] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:14:12.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:12.917934 8407 raft_consensus.cc:504] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:14:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:12.920465 8407 leader_election.cc:288] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805), 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947)
20:14:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:12.923229 8147 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:14:12.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:12.926896 8146 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947): Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:14:12.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:12.927879 8147 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805): Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111)
20:14:12.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:12.928442 8147 leader_election.cc:302] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f51614eb31644f6f91fa39a6c96fcaf0; no voters: 122b53cc6dab4daa8dc015f29f856547, bbcc3cd8263042c79af252ac96efca5d
20:14:12.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:12.929344 8407 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:14:13.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:13.158174 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:13.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:13.158540 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:14.050 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:14.052 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:14.054 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:14.054 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:14.054 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:14.054 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:14.054 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:14.054 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:14.057 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:14.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:14.162858 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:14.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:14.163182 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:14.810 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:14.812 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:14.814 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:14.815 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:14.816 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:14.816 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:14.816 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:14.816 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:14.817 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:14.835807 6906 debug-util.cc:398] Leaking SignalData structure 0x7b080009b0e0 after lost signal to thread 6824
20:14:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:14.836711 6906 debug-util.cc:398] Leaking SignalData structure 0x7b080003f2e0 after lost signal to thread 6911
20:14:14.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:14.986784 8420 raft_consensus.cc:482] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:14:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:14.987327 8420 raft_consensus.cc:504] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:14:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:14.989552 8420 leader_election.cc:288] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805), 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947)
20:14:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:14.995400 8147 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805): Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111)
20:14:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:14.995947 8146 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947): Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:14:14.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:14.996503 8146 leader_election.cc:302] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f51614eb31644f6f91fa39a6c96fcaf0; no voters: 122b53cc6dab4daa8dc015f29f856547, bbcc3cd8263042c79af252ac96efca5d
20:14:14.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:14.997706 8420 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:14:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:15.168314 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:15.168754 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:16.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:16.172165 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:16.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:16.172490 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:14:16 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133656, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:14:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:16.713835 8134 init.cc:295] Successfully reacquired a new kerberos TGT
20:14:17.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:17.131924 8114 master_runner.cc:186] Error getting master registration for 127.7.109.254:39805: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111),
20:14:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:17.132819 8114 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:14:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:17.178782 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:17.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:17.180263 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:18.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:18.186800 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:18.187436 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:18.718 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:18.730 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:18.787 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:18.792 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:18.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:18.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:18.801 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:18.801 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:18.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:19.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:19.193456 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:19.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:19.194096 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:19.429097 8439 raft_consensus.cc:482] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:14:19.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:19.429886 8439 raft_consensus.cc:504] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbcc3cd8263042c79af252ac96efca5d" member_type: VOTER last_known_addr { host: "127.7.109.254" port: 39805 } } peers { permanent_uuid: "122b53cc6dab4daa8dc015f29f856547" member_type: VOTER last_known_addr { host: "127.7.109.253" port: 35947 } } peers { permanent_uuid: "f51614eb31644f6f91fa39a6c96fcaf0" member_type: VOTER last_known_addr { host: "127.7.109.252" port: 36293 } }
20:14:19.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:19.432873 8439 leader_election.cc:288] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805), 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947)
20:14:19.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:19.436645 8147 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:14:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:19.442960 8147 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbcc3cd8263042c79af252ac96efca5d (127.7.109.254:39805): Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111)
20:14:19.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:19.444891 8146 leader_election.cc:334] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 122b53cc6dab4daa8dc015f29f856547 (127.7.109.253:35947): Network error: Client connection negotiation failed: client connection to 127.7.109.253:35947: connect: Connection refused (error 111)
20:14:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:19.445542 8146 leader_election.cc:302] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f51614eb31644f6f91fa39a6c96fcaf0; no voters: 122b53cc6dab4daa8dc015f29f856547, bbcc3cd8263042c79af252ac96efca5d
20:14:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:19.447118 8439 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:14:19.950 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:19.952 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:19.954 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:19.954 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:19.955 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:19.955 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:19.955 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:19.956 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:19.957 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:20.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:20.197487 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:20.197800 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:20.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:20.325956 7184 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae2c0 after lost signal to thread 7001
20:14:20.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:20.327263 7184 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7ec0 after lost signal to thread 7187
20:14:21.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:21.202100 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:21.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:21.202467 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:22.030 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:22.034 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:22.034 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:22.035 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:22.038 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:22.041 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:22.041 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:22.041 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:22.043 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:22.136479 8114 master_runner.cc:186] Error getting master registration for 127.7.109.254:39805: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111),
20:14:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:22.137081 8114 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:14:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:22.205875 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:22.206241 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:23.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:23.209668 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:23.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:23.210002 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:24.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:24.213685 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:24.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:24.214082 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:24.970 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:24.971 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:24.973 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:24.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:24.974 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:24.974 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:24.974 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:24.974 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:24.977 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:25.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:25.219216 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:25.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:25.219671 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:25.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:25.915194 7583 debug-util.cc:398] Leaking SignalData structure 0x7b08000bdce0 after lost signal to thread 7401
20:14:25.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:25.916738 7583 debug-util.cc:398] Leaking SignalData structure 0x7b08000aee00 after lost signal to thread 7586
20:14:26.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:26.224797 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:26.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:26.225244 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 20:14:26 dist-test-slave-kvfs krb5kdc[7625](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786133666, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.109.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:14:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:26.717578 8134 init.cc:295] Successfully reacquired a new kerberos TGT
20:14:27.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:27.141810 8114 master_runner.cc:186] Error getting master registration for 127.7.109.254:39805: Network error: Client connection negotiation failed: client connection to 127.7.109.254:39805: connect: Connection refused (error 111),
20:14:27.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:27.142516 8114 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:14:27.210 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.254:39805(127.7.109.254:39805)] connecting to peer
20:14:27.212 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.109.253:35947(127.7.109.253:35947)] connecting to peer
20:14:27.213 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:27.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.254:39805(127.7.109.254:39805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:27.213 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.254:39805: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805
20:14:27.214 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:27.215 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.109.253:35947(127.7.109.253:35947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:27.215 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.109.253:35947: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
20:14:27.217 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947; will retry
20:14:27.225 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28070), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [234ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [249ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [654ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [684ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [686ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [691ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [692ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [705ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [722ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [723ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [726ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [762ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [764ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [764ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [766ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [782ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [782ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [784ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [816ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [823ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [824ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [825ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [936ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [944ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [946ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [948ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [951ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1136ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [1149ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1150ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1152ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [1153ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1576ms] refreshing cache from master, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [1578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [1579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1584ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1585ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [1586ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [2496ms] refreshing cache from master, [2497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [2499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [2500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [2502ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [2504ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [2505ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [2505ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [3216ms] refreshing cache from master, [3216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [3218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [3219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [3222ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [3224ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [3225ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [3226ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [6256ms] refreshing cache from master, [6257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [6258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [6259ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [6262ms] too many traces: truncated at 100 traces, deferred=Deferred@673576344(state=PENDING, result=null, callback=(continuation of Deferred@536153234 after retry RPC after error@291846785) -> (continuation of Deferred@1786461585 after retry RPC after error@1836424307) -> (continuation of Deferred@1515429075 after retry RPC after error@1650066474) -> (continuation of Deferred@1375611461 after retry RPC after error@279556621) -> (continuation of Deferred@1448743375 after retry RPC after error@1687364730) -> (continuation of Deferred@1151305391 after retry RPC after error@427523563) -> (continuation of Deferred@531219298 after retry RPC after error@1659143193) -> (continuation of Deferred@1517734932 after retry RPC after error@404776596) -> (continuation of Deferred@1237517345 after retry RPC after error@264340812) -> (continuation of Deferred@1673260748 after retry RPC after error@2025070825) -> (continuation of Deferred@688378322 after retry RPC after error@711969371) -> (continuation of Deferred@379489364 after retry RPC after error@1333024006) -> (continuation of Deferred@1483379835 after retry RPC after error@201564223) -> (continuation of Deferred@1847419335 after retry RPC after error@53269863) -> (continuation of Deferred@258555957 after retry RPC after error@519707583) -> (continuation of Deferred@460079870 after retry RPC after error@2114857753) -> (continuation of Deferred@992195589 after retry RPC after error@1967810365) -> (continuation of Deferred@206090456 after retry RPC after error@59839659) -> (continuation of Deferred@632556352 after retry RPC after error@56746820) -> (continuation of Deferred@1248222592 after retry RPC after error@1104755755) -> (continuation of Deferred@1144181083 after retry RPC after error@1212506458) -> (continuation of Deferred@808959887 after retry RPC after error@948434374), errback=(continuation of Deferred@536153234 after retry RPC after error@291846785) -> (continuation of Deferred@1786461585 after retry RPC after error@1836424307) -> (continuation of Deferred@1515429075 after retry RPC after error@1650066474) -> (continuation of Deferred@1375611461 after retry RPC after error@279556621) -> (continuation of Deferred@1448743375 after retry RPC after error@1687364730) -> (continuation of Deferred@1151305391 after retry RPC after error@427523563) -> (continuation of Deferred@531219298 after retry RPC after error@1659143193) -> (continuation of Deferred@1517734932 after retry RPC after error@404776596) -> (continuation of Deferred@1237517345 after retry RPC after error@264340812) -> (continuation of Deferred@1673260748 after retry RPC after error@2025070825) -> (continuation of Deferred@688378322 after retry RPC after error@711969371) -> (continuation of Deferred@379489364 after retry RPC after error@1333024006) -> (continuation of Deferred@1483379835 after retry RPC after error@201564223) -> (continuation of Deferred@1847419335 after retry RPC after error@53269863) -> (continuation of Deferred@258555957 after retry RPC after error@519707583) -> (continuation of Deferred@460079870 after retry RPC after error@2114857753) -> (continuation of Deferred@992195589 after retry RPC after error@1967810365) -> (continuation of Deferred@206090456 after retry RPC after error@59839659) -> (continuation of Deferred@632556352 after retry RPC after error@56746820) -> (continuation of Deferred@1248222592 after retry RPC after error@1104755755) -> (continuation of Deferred@1144181083 after retry RPC after error@1212506458) -> (continuation of Deferred@808959887 after retry RPC after error@948434374)))
20:14:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:27.229799 8238 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f51614eb31644f6f91fa39a6c96fcaf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:14:27.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:27.230859 8238 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:14:27.234 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.109.254:39805(127.7.109.254:39805)] disconnecting while in state TERMINATED
20:14:27.235 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.109.252:36293(127.7.109.252:36293)] disconnecting while in state READY
20:14:27.236 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.109.253:35947(127.7.109.253:35947)] disconnecting while in state TERMINATED
20:14:27.256 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:14:27.268 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:14:27.279 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.7.109.252:36293(127.7.109.252:36293)] handling channelInactive
20:14:27.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:27.281740 7607 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu with pid 8114
20:14:27.283 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.109.252:36293(127.7.109.252:36293)] cleaning up while in state READY due to: connection closed
20:14:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T20:14:28Z chronyd exiting
20:14:28.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:28.665244 6807 debug-util.cc:398] Leaking SignalData structure 0x7b080009cc20 after lost signal to thread 6728
20:14:28.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:28.665961 6807 debug-util.cc:398] Leaking SignalData structure 0x7b080009cbe0 after lost signal to thread 6810
20:14:29.726 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:14:29.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:29.728854 6611 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu with pid 6995
20:14:29.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:29.804425 6611 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu with pid 7199
20:14:29.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:29.856753 6611 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu with pid 7398
20:14:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:29.914315 6611 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu with pid 6629
20:14:29.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:29.976434 6611 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu with pid 6723
20:14:29.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 20:14:29.976291 6752 connection.cc:535] client connection to 127.6.116.254:40471 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:14:30.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 20:14:30.054284 6611 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasknE2NB6/build/tsan/bin/kudu with pid 6816
20:14:31.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T20:14:31Z chronyd exiting
20:14:32.341 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28070), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [234ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [249ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [654ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [684ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [686ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [691ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [692ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [705ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [722ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [723ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [726ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [762ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [764ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [764ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [766ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [782ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [782ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [784ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [816ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [823ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [824ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [825ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [936ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [944ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [946ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [948ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [951ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1136ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [1149ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1150ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1152ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [1153ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1576ms] refreshing cache from master, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [1578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [1579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1584ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1585ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [1586ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [2496ms] refreshing cache from master, [2497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [2499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [2500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [2502ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [2504ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [2505ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [2505ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [3216ms] refreshing cache from master, [3216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [3218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [3219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [3222ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [3224ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [3225ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [3226ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [6256ms] refreshing cache from master, [6257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [6258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [6259ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [6262ms] too many traces: truncated at 100 traces, deferred=Deferred@673576344(state=PENDING, result=null, callback=(continuation of Deferred@536153234 after retry RPC after error@291846785) -> (continuation of Deferred@1786461585 after retry RPC after error@1836424307) -> (continuation of Deferred@1515429075 after retry RPC after error@1650066474) -> (continuation of Deferred@1375611461 after retry RPC after error@279556621) -> (continuation of Deferred@1448743375 after retry RPC after error@1687364730) -> (continuation of Deferred@1151305391 after retry RPC after error@427523563) -> (continuation of Deferred@531219298 after retry RPC after error@1659143193) -> (continuation of Deferred@1517734932 after retry RPC after error@404776596) -> (continuation of Deferred@1237517345 after retry RPC after error@264340812) -> (continuation of Deferred@1673260748 after retry RPC after error@2025070825) -> (continuation of Deferred@688378322 after retry RPC after error@711969371) -> (continuation of Deferred@379489364 after retry RPC after error@1333024006) -> (continuation of Deferred@1483379835 after retry RPC after error@201564223) -> (continuation of Deferred@1847419335 after retry RPC after error@53269863) -> (continuation of Deferred@258555957 after retry RPC after error@519707583) -> (continuation of Deferred@460079870 after retry RPC after error@2114857753) -> (continuation of Deferred@992195589 after retry RPC after error@1967810365) -> (continuation of Deferred@206090456 after retry RPC after error@59839659) -> (continuation of Deferred@632556352 after retry RPC after error@56746820) -> (continuation of Deferred@1248222592 after retry RPC after error@1104755755) -> (continuation of Deferred@1144181083 after retry RPC after error@1212506458) -> (continuation of Deferred@808959887 after retry RPC after error@948434374), errback=(continuation of Deferred@536153234 after retry RPC after error@291846785) -> (continuation of Deferred@1786461585 after retry RPC after error@1836424307) -> (continuation of Deferred@1515429075 after retry RPC after error@1650066474) -> (continuation of Deferred@1375611461 after retry RPC after error@279556621) -> (continuation of Deferred@1448743375 after retry RPC after error@1687364730) -> (continuation of Deferred@1151305391 after retry RPC after error@427523563) -> (continuation of Deferred@531219298 after retry RPC after error@1659143193) -> (continuation of Deferred@1517734932 after retry RPC after error@404776596) -> (continuation of Deferred@1237517345 after retry RPC after error@264340812) -> (continuation of Deferred@1673260748 after retry RPC after error@2025070825) -> (continuation of Deferred@688378322 after retry RPC after error@711969371) -> (continuation of Deferred@379489364 after retry RPC after error@1333024006) -> (continuation of Deferred@1483379835 after retry RPC after error@201564223) -> (continuation of Deferred@1847419335 after retry RPC after error@53269863) -> (continuation of Deferred@258555957 after retry RPC after error@519707583) -> (continuation of Deferred@460079870 after retry RPC after error@2114857753) -> (continuation of Deferred@992195589 after retry RPC after error@1967810365) -> (continuation of Deferred@206090456 after retry RPC after error@59839659) -> (continuation of Deferred@632556352 after retry RPC after error@56746820) -> (continuation of Deferred@1248222592 after retry RPC after error@1104755755) -> (continuation of Deferred@1144181083 after retry RPC after error@1212506458) -> (continuation of Deferred@808959887 after retry RPC after error@948434374)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.109.253:35947
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28070), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [234ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [249ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [654ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [684ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [686ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [691ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [692ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [705ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [722ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [723ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [726ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [762ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [764ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [764ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [766ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [782ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [782ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [784ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [816ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [823ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [824ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [825ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [936ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [944ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [946ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [948ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [951ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1136ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [1149ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1150ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1152ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [1153ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1576ms] refreshing cache from master, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [1578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [1579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1584ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1585ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [1586ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [2496ms] refreshing cache from master, [2497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [2499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [2500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [2502ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [2504ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [2505ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [2505ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [3216ms] refreshing cache from master, [3216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [3218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [3219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [3222ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [3224ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [3225ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [3226ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [6256ms] refreshing cache from master, [6257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [6258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [6259ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [6262ms] too many traces: truncated at 100 traces, deferred=Deferred@673576344(state=PENDING, result=null, callback=(continuation of Deferred@536153234 after retry RPC after error@291846785) -> (continuation of Deferred@1786461585 after retry RPC after error@1836424307) -> (continuation of Deferred@1515429075 after retry RPC after error@1650066474) -> (continuation of Deferred@1375611461 after retry RPC after error@279556621) -> (continuation of Deferred@1448743375 after retry RPC after error@1687364730) -> (continuation of Deferred@1151305391 after retry RPC after error@427523563) -> (continuation of Deferred@531219298 after retry RPC after error@1659143193) -> (continuation of Deferred@1517734932 after retry RPC after error@404776596) -> (continuation of Deferred@1237517345 after retry RPC after error@264340812) -> (continuation of Deferred@1673260748 after retry RPC after error@2025070825) -> (continuation of Deferred@688378322 after retry RPC after error@711969371) -> (continuation of Deferred@379489364 after retry RPC after error@1333024006) -> (continuation of Deferred@1483379835 after retry RPC after error@201564223) -> (continuation of Deferred@1847419335 after retry RPC after error@53269863) -> (continuation of Deferred@258555957 after retry RPC after error@519707583) -> (continuation of Deferred@460079870 after retry RPC after error@2114857753) -> (continuation of Deferred@992195589 after retry RPC after error@1967810365) -> (continuation of Deferred@206090456 after retry RPC after error@59839659) -> (continuation of Deferred@632556352 after retry RPC after error@56746820) -> (continuation of Deferred@1248222592 after retry RPC after error@1104755755) -> (continuation of Deferred@1144181083 after retry RPC after error@1212506458) -> (continuation of Deferred@808959887 after retry RPC after error@948434374), errback=(continuation of Deferred@536153234 after retry RPC after error@291846785) -> (continuation of Deferred@1786461585 after retry RPC after error@1836424307) -> (continuation of Deferred@1515429075 after retry RPC after error@1650066474) -> (continuation of Deferred@1375611461 after retry RPC after error@279556621) -> (continuation of Deferred@1448743375 after retry RPC after error@1687364730) -> (continuation of Deferred@1151305391 after retry RPC after error@427523563) -> (continuation of Deferred@531219298 after retry RPC after error@1659143193) -> (continuation of Deferred@1517734932 after retry RPC after error@404776596) -> (continuation of Deferred@1237517345 after retry RPC after error@264340812) -> (continuation of Deferred@1673260748 after retry RPC after error@2025070825) -> (continuation of Deferred@688378322 after retry RPC after error@711969371) -> (continuation of Deferred@379489364 after retry RPC after error@1333024006) -> (continuation of Deferred@1483379835 after retry RPC after error@201564223) -> (continuation of Deferred@1847419335 after retry RPC after error@53269863) -> (continuation of Deferred@258555957 after retry RPC after error@519707583) -> (continuation of Deferred@460079870 after retry RPC after error@2114857753) -> (continuation of Deferred@992195589 after retry RPC after error@1967810365) -> (continuation of Deferred@206090456 after retry RPC after error@59839659) -> (continuation of Deferred@632556352 after retry RPC after error@56746820) -> (continuation of Deferred@1248222592 after retry RPC after error@1104755755) -> (continuation of Deferred@1144181083 after retry RPC after error@1212506458) -> (continuation of Deferred@808959887 after retry RPC after error@948434374)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.109.253:35947
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
20:14:32.762 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:14:32.762 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28070), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [234ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [249ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [654ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [684ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [686ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [691ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [692ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [705ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [722ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [723ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [726ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [762ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [764ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [764ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [766ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [782ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [782ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [784ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [816ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [823ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [824ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [825ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [936ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [944ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [946ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [948ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [951ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1136ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [1149ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1150ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1152ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [1153ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1576ms] refreshing cache from master, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [1578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [1579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1584ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1585ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [1586ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [2496ms] refreshing cache from master, [2497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [2499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [2500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [2502ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [2504ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [2505ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [2505ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [3216ms] refreshing cache from master, [3216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [3218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [3219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [3222ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [3224ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [3225ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [3226ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [6256ms] refreshing cache from master, [6257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [6258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [6259ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [6262ms] too many traces: truncated at 100 traces, deferred=Deferred@673576344(state=PENDING, result=null, callback=(continuation of Deferred@536153234 after retry RPC after error@291846785) -> (continuation of Deferred@1786461585 after retry RPC after error@1836424307) -> (continuation of Deferred@1515429075 after retry RPC after error@1650066474) -> (continuation of Deferred@1375611461 after retry RPC after error@279556621) -> (continuation of Deferred@1448743375 after retry RPC after error@1687364730) -> (continuation of Deferred@1151305391 after retry RPC after error@427523563) -> (continuation of Deferred@531219298 after retry RPC after error@1659143193) -> (continuation of Deferred@1517734932 after retry RPC after error@404776596) -> (continuation of Deferred@1237517345 after retry RPC after error@264340812) -> (continuation of Deferred@1673260748 after retry RPC after error@2025070825) -> (continuation of Deferred@688378322 after retry RPC after error@711969371) -> (continuation of Deferred@379489364 after retry RPC after error@1333024006) -> (continuation of Deferred@1483379835 after retry RPC after error@201564223) -> (continuation of Deferred@1847419335 after retry RPC after error@53269863) -> (continuation of Deferred@258555957 after retry RPC after error@519707583) -> (continuation of Deferred@460079870 after retry RPC after error@2114857753) -> (continuation of Deferred@992195589 after retry RPC after error@1967810365) -> (continuation of Deferred@206090456 after retry RPC after error@59839659) -> (continuation of Deferred@632556352 after retry RPC after error@56746820) -> (continuation of Deferred@1248222592 after retry RPC after error@1104755755) -> (continuation of Deferred@1144181083 after retry RPC after error@1212506458) -> (continuation of Deferred@808959887 after retry RPC after error@948434374), errback=(continuation of Deferred@536153234 after retry RPC after error@291846785) -> (continuation of Deferred@1786461585 after retry RPC after error@1836424307) -> (continuation of Deferred@1515429075 after retry RPC after error@1650066474) -> (continuation of Deferred@1375611461 after retry RPC after error@279556621) -> (continuation of Deferred@1448743375 after retry RPC after error@1687364730) -> (continuation of Deferred@1151305391 after retry RPC after error@427523563) -> (continuation of Deferred@531219298 after retry RPC after error@1659143193) -> (continuation of Deferred@1517734932 after retry RPC after error@404776596) -> (continuation of Deferred@1237517345 after retry RPC after error@264340812) -> (continuation of Deferred@1673260748 after retry RPC after error@2025070825) -> (continuation of Deferred@688378322 after retry RPC after error@711969371) -> (continuation of Deferred@379489364 after retry RPC after error@1333024006) -> (continuation of Deferred@1483379835 after retry RPC after error@201564223) -> (continuation of Deferred@1847419335 after retry RPC after error@53269863) -> (continuation of Deferred@258555957 after retry RPC after error@519707583) -> (continuation of Deferred@460079870 after retry RPC after error@2114857753) -> (continuation of Deferred@992195589 after retry RPC after error@1967810365) -> (continuation of Deferred@206090456 after retry RPC after error@59839659) -> (continuation of Deferred@632556352 after retry RPC after error@56746820) -> (continuation of Deferred@1248222592 after retry RPC after error@1104755755) -> (continuation of Deferred@1144181083 after retry RPC after error@1212506458) -> (continuation of Deferred@808959887 after retry RPC after error@948434374)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.109.253:35947
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28070), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [234ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [249ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [654ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [684ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [686ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [691ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [692ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [705ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [722ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [723ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [726ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [762ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [764ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [764ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [766ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [782ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [782ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [784ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [816ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [823ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [824ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [825ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [936ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [944ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [946ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [948ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [951ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1136ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [1149ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1150ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1152ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [1153ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1576ms] refreshing cache from master, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [1578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [1579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [1584ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [1585ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [1586ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [2496ms] refreshing cache from master, [2497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [2499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [2500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [2502ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [2504ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [2505ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [2505ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [3216ms] refreshing cache from master, [3216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [3218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [3219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [3222ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.253:35947: Network error: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947, [3224ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.254:39805: Network error: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [3225ms] Sub RPC ConnectToMaster: received response from server master-127.7.109.252:36293: OK, [3226ms] delaying RPC due to: Service unavailable: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805, [6256ms] refreshing cache from master, [6257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.254:39805, [6258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.252:36293, [6259ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.109.253:35947, [6262ms] too many traces: truncated at 100 traces, deferred=Deferred@673576344(state=PENDING, result=null, callback=(continuation of Deferred@536153234 after retry RPC after error@291846785) -> (continuation of Deferred@1786461585 after retry RPC after error@1836424307) -> (continuation of Deferred@1515429075 after retry RPC after error@1650066474) -> (continuation of Deferred@1375611461 after retry RPC after error@279556621) -> (continuation of Deferred@1448743375 after retry RPC after error@1687364730) -> (continuation of Deferred@1151305391 after retry RPC after error@427523563) -> (continuation of Deferred@531219298 after retry RPC after error@1659143193) -> (continuation of Deferred@1517734932 after retry RPC after error@404776596) -> (continuation of Deferred@1237517345 after retry RPC after error@264340812) -> (continuation of Deferred@1673260748 after retry RPC after error@2025070825) -> (continuation of Deferred@688378322 after retry RPC after error@711969371) -> (continuation of Deferred@379489364 after retry RPC after error@1333024006) -> (continuation of Deferred@1483379835 after retry RPC after error@201564223) -> (continuation of Deferred@1847419335 after retry RPC after error@53269863) -> (continuation of Deferred@258555957 after retry RPC after error@519707583) -> (continuation of Deferred@460079870 after retry RPC after error@2114857753) -> (continuation of Deferred@992195589 after retry RPC after error@1967810365) -> (continuation of Deferred@206090456 after retry RPC after error@59839659) -> (continuation of Deferred@632556352 after retry RPC after error@56746820) -> (continuation of Deferred@1248222592 after retry RPC after error@1104755755) -> (continuation of Deferred@1144181083 after retry RPC after error@1212506458) -> (continuation of Deferred@808959887 after retry RPC after error@948434374), errback=(continuation of Deferred@536153234 after retry RPC after error@291846785) -> (continuation of Deferred@1786461585 after retry RPC after error@1836424307) -> (continuation of Deferred@1515429075 after retry RPC after error@1650066474) -> (continuation of Deferred@1375611461 after retry RPC after error@279556621) -> (continuation of Deferred@1448743375 after retry RPC after error@1687364730) -> (continuation of Deferred@1151305391 after retry RPC after error@427523563) -> (continuation of Deferred@531219298 after retry RPC after error@1659143193) -> (continuation of Deferred@1517734932 after retry RPC after error@404776596) -> (continuation of Deferred@1237517345 after retry RPC after error@264340812) -> (continuation of Deferred@1673260748 after retry RPC after error@2025070825) -> (continuation of Deferred@688378322 after retry RPC after error@711969371) -> (continuation of Deferred@379489364 after retry RPC after error@1333024006) -> (continuation of Deferred@1483379835 after retry RPC after error@201564223) -> (continuation of Deferred@1847419335 after retry RPC after error@53269863) -> (continuation of Deferred@258555957 after retry RPC after error@519707583) -> (continuation of Deferred@460079870 after retry RPC after error@2114857753) -> (continuation of Deferred@992195589 after retry RPC after error@1967810365) -> (continuation of Deferred@206090456 after retry RPC after error@59839659) -> (continuation of Deferred@632556352 after retry RPC after error@56746820) -> (continuation of Deferred@1248222592 after retry RPC after error@1104755755) -> (continuation of Deferred@1144181083 after retry RPC after error@1212506458) -> (continuation of Deferred@808959887 after retry RPC after error@948434374)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.109.254:39805,127.7.109.252:36293,127.7.109.253:35947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.254:39805(127.7.109.254:39805): Connection refused: /127.7.109.254:39805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.109.253:35947(127.7.109.253:35947): Connection refused: /127.7.109.253:35947
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.109.253:35947
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)