06:06:16.649 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:06:16.666 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoDNYHM/test-tmp
06:06:16.669 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:06:16.673 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoDNYHM/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:06:16.973 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
06:06:17.229 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:06:17.231 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:06:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:17.236960 1678 mini_oidc.cc:152] Starting JWKS server
06:06:17.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:17.243544 1678 webserver.cc:460] Webserver started at http://127.0.0.1:45027/ using document root <none> and password file <none>
06:06:17.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:17.244333 1678 mini_oidc.cc:189] Starting OIDC Discovery server
06:06:17.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:17.249590 1678 webserver.cc:460] Webserver started at http://127.0.0.1:45141/ using document root <none> and password file <none>
06:06:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T06:06:17Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:06:17.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T06:06:17Z Disabled control of system clock
06:06:17.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:17.320905 1678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:17.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/wal
06:06:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/data
06:06:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/logs
06:06:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/data/info.pb
06:06:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:06:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.190:44249
06:06:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.190
06:06:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json with env {}
06:06:17.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.775609 1690 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:17.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.776173 1690 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:17.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.776780 1690 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:17.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.807222 1690 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:06:17.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.807482 1690 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.807683 1690 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:06:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.807928 1690 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:06:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.808228 1690 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:06:17.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.808490 1690 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45027/jwks.json
06:06:17.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:17.843559 1690 master_runner.cc:386] Master server non-default flags:
06:06:17.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:17.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:17.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/data
06:06:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/wal
06:06:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.190:44249
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/data/info.pb
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.190
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/logs
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:17.845585 1690 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:17.848037 1690 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.887228 1697 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:19.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.888254 1699 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:19.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:17.887372 1701 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:19.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:19.302706 1700 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1412 milliseconds
06:06:19.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.302860 1690 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:19.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.304545 1690 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.309420 1690 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.310804 1690 hybrid_clock.cc:648] HybridClock initialized: now 1786428379310749 us; error 66 us; skew 500 ppm
06:06:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.311972 1690 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:19.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.324915 1690 webserver.cc:460] Webserver started at http://127.1.163.190:36661/ using document root <none> and password file <none>
06:06:19.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.326359 1690 fs_manager.cc:359] Metadata directory not provided
06:06:19.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.326623 1690 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:19.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.327368 1690 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:06:19.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.333693 1690 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/data/instance:
06:06:19.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f9babe013fa4875adc71cf23633e9d1"
06:06:19.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:19 on dist-test-slave-pgkr"
06:06:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.335438 1690 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/wal/instance:
06:06:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f9babe013fa4875adc71cf23633e9d1"
06:06:19.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:19 on dist-test-slave-pgkr"
06:06:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.346760 1690 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.006s sys 0.005s
06:06:19.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.357805 1708 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:19.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.359647 1690 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.003s sys 0.003s
06:06:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.360181 1690 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/wal
06:06:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f9babe013fa4875adc71cf23633e9d1"
06:06:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:19 on dist-test-slave-pgkr"
06:06:19.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.360615 1690 fs_report.cc:383] FS layout report
06:06:19.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:19.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/wal
06:06:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/wal
06:06:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/data/data
06:06:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:19.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:19.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:19.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:19.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:19.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:19.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:19.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:19.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.447654 1690 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:19.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.450459 1690 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:19.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.451231 1690 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:19.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.723594 1690 rpc_server.cc:307] RPC server started. Bound to: 127.1.163.190:44249
06:06:19.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.723816 1767 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.163.190:44249 every 8 connection(s)
06:06:19.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.727439 1690 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/data/info.pb
06:06:19.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.737093 1678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 1690
06:06:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.737928 1678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-0/wal/instance
06:06:19.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.744012 1768 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.745566 1678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/wal
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/data
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/logs
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/data/info.pb
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:06:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:19.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.189:33127
06:06:19.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.189
06:06:19.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:19.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:19.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:19.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:19.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:19.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:19.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json with env {}
06:06:19.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.771991 1768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } has no permanent_uuid. Determining permanent_uuid...
06:06:19.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.837603 1768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } has no permanent_uuid. Determining permanent_uuid...
06:06:19.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:19.841761 1709 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.163.189:33127: connect: Connection refused (error 111)
06:06:19.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:19.853266 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.189:33127: Network error: Client connection negotiation failed: client connection to 127.1.163.189:33127: connect: Connection refused (error 111)
06:06:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.902812 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } attempt: 1
06:06:19.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:19.909641 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.189:33127: Network error: Client connection negotiation failed: client connection to 127.1.163.189:33127: connect: Connection refused (error 111)
06:06:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:19.978230 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } attempt: 2
06:06:19.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:19.985045 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.189:33127: Network error: Client connection negotiation failed: client connection to 127.1.163.189:33127: connect: Connection refused (error 111)
06:06:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:20.076551 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } attempt: 3
06:06:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.084250 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.189:33127: Network error: Client connection negotiation failed: client connection to 127.1.163.189:33127: connect: Connection refused (error 111)
06:06:20.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:20.227886 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } attempt: 4
06:06:20.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.234969 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.189:33127: Network error: Client connection negotiation failed: client connection to 127.1.163.189:33127: connect: Connection refused (error 111)
06:06:20.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.283784 1770 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:20.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.284602 1770 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:20.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.285387 1770 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:20.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.317267 1770 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:06:20.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.317538 1770 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:20.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.317801 1770 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:06:20.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.317999 1770 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:06:20.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.318184 1770 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:06:20.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.318326 1770 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45027/jwks.json
06:06:20.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:20.353489 1770 master_runner.cc:386] Master server non-default flags:
06:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/data
06:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/wal
06:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.189:33127
06:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json
06:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/data/info.pb
06:06:20.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.189
06:06:20.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:20.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/logs
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:20.354882 1770 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:20.356519 1770 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:20.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.391865 1781 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:20.534737 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } attempt: 5
06:06:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.541633 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.189:33127: Network error: Client connection negotiation failed: client connection to 127.1.163.189:33127: connect: Connection refused (error 111)
06:06:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.089162 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } attempt: 6
06:06:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:21.103636 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.189:33127: Network error: Client connection negotiation failed: client connection to 127.1.163.189:33127: connect: Connection refused (error 111)
06:06:21.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.391855 1780 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:21.717561 1782 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:06:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.717687 1770 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:20.393333 1783 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.723749 1770 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:21.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.728384 1770 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:21.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.729910 1770 hybrid_clock.cc:648] HybridClock initialized: now 1786428381729879 us; error 53 us; skew 500 ppm
06:06:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.731447 1770 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:21.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.748365 1770 webserver.cc:460] Webserver started at http://127.1.163.189:34803/ using document root <none> and password file <none>
06:06:21.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.749410 1770 fs_manager.cc:359] Metadata directory not provided
06:06:21.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.749660 1770 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:21.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.750231 1770 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:06:21.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.755862 1770 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/data/instance:
06:06:21.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5ed02a9beb442d7b909c6236c52a184"
06:06:21.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:21 on dist-test-slave-pgkr"
06:06:21.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.757189 1770 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/wal/instance:
06:06:21.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5ed02a9beb442d7b909c6236c52a184"
06:06:21.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:21 on dist-test-slave-pgkr"
06:06:21.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.766852 1770 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
06:06:21.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.775545 1792 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.776652 1770 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.008s sys 0.000s
06:06:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.777016 1770 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/wal
06:06:21.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5ed02a9beb442d7b909c6236c52a184"
06:06:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:21 on dist-test-slave-pgkr"
06:06:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.777382 1770 fs_report.cc:383] FS layout report
06:06:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/wal
06:06:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/wal
06:06:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/data/data
06:06:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:21.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.824295 1770 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.826555 1770 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:21.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:21.827391 1770 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:22.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.021479 1770 rpc_server.cc:307] RPC server started. Bound to: 127.1.163.189:33127
06:06:22.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.021606 1845 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.163.189:33127 every 8 connection(s)
06:06:22.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.025149 1770 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/data/info.pb
06:06:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.029455 1678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 1770
06:06:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.030094 1678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-1/wal/instance
06:06:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.034794 1678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/wal
06:06:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/data
06:06:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/logs
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/data/info.pb
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:06:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:22.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:22.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.188:35501
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.188
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json with env {}
06:06:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.037359 1846 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:06:22.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.063575 1846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } has no permanent_uuid. Determining permanent_uuid...
06:06:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.109630 1846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } has no permanent_uuid. Determining permanent_uuid...
06:06:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.145210 1846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } has no permanent_uuid. Determining permanent_uuid...
06:06:22.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.149428 1796 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.160553 1846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.164199 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } attempt: 7
06:06:22.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.195717 1768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } has no permanent_uuid. Determining permanent_uuid...
06:06:22.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.208709 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.210188 1846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 1
06:06:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.215896 1846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.267251 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 1
06:06:22.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.273519 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.284487 1846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 2
06:06:22.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.291016 1846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.354987 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 2
06:06:22.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.361462 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.382469 1846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 3
06:06:22.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.388460 1846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.447014 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 3
06:06:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.453217 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.531852 1846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 4
06:06:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.537537 1846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.593712 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 4
06:06:22.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.599351 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.640447 1848 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.641115 1848 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.641834 1848 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.677136 1848 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.677423 1848 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.677623 1848 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:06:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.677829 1848 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.678014 1848 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:06:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.678159 1848 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45027/jwks.json
06:06:22.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.722370 1848 master_runner.cc:386] Master server non-default flags:
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/data
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/wal
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.188:35501
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/data/info.pb
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.188
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/logs
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.724012 1848 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:22.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.726251 1848 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:22.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.773818 1862 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:22.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.836990 1846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 5
06:06:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.843070 1846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:22.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:22.883038 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 5
06:06:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.889703 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:23.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:23.390769 1846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 6
06:06:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:23.406533 1846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:23.442232 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 6
06:06:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:23.460883 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:24.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.773936 1867 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:24.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:22.776005 1865 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:24.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:24.158555 1866 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:06:24.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.158764 1848 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:24.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.165948 1848 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:24.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.170702 1848 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.172138 1848 hybrid_clock.cc:648] HybridClock initialized: now 1786428384172031 us; error 128 us; skew 500 ppm
06:06:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.173054 1848 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:24.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.187460 1848 webserver.cc:460] Webserver started at http://127.1.163.188:42655/ using document root <none> and password file <none>
06:06:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.189294 1848 fs_manager.cc:359] Metadata directory not provided
06:06:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.189543 1848 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.190189 1848 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:06:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.195861 1848 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/data/instance:
06:06:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "674cc974e7ad43d1b92066aa374f8648"
06:06:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:24 on dist-test-slave-pgkr"
06:06:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.197134 1848 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/wal/instance:
06:06:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "674cc974e7ad43d1b92066aa374f8648"
06:06:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:24 on dist-test-slave-pgkr"
06:06:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.210165 1848 fs_manager.cc:691] Time spent creating directory manager: real 0.012s user 0.005s sys 0.009s
06:06:24.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.221024 1881 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:24.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.222239 1848 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.004s sys 0.005s
06:06:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.222646 1848 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/wal
06:06:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "674cc974e7ad43d1b92066aa374f8648"
06:06:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:24 on dist-test-slave-pgkr"
06:06:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.223096 1848 fs_report.cc:383] FS layout report
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/wal
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/wal
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/data/data
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:24.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.275094 1848 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:24.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.277455 1848 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:24.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.278496 1848 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:24.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.467399 1846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 7
06:06:24.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:24.485214 1846 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:24.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.494429 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 7
06:06:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:24.506847 1768 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.163.188:35501: Network error: Client connection negotiation failed: client connection to 127.1.163.188:35501: connect: Connection refused (error 111)
06:06:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.533944 1848 rpc_server.cc:307] RPC server started. Bound to: 127.1.163.188:35501
06:06:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.534075 1942 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.163.188:35501 every 8 connection(s)
06:06:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.538880 1848 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/data/info.pb
06:06:24.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.547626 1678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 1848
06:06:24.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.548172 1678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/master-2/wal/instance
06:06:24.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.549471 1943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:06:24.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.582294 1943 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } has no permanent_uuid. Determining permanent_uuid...
06:06:24.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.628652 1943 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } has no permanent_uuid. Determining permanent_uuid...
06:06:24.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.646867 1943 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } has no permanent_uuid. Determining permanent_uuid...
06:06:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.679621 1943 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648: Bootstrap starting.
06:06:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.697118 1943 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648: Neither blocks nor log segments found. Creating new log.
06:06:24.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.702491 1943 log.cc:824] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648: Log is configured to *not* fsync() on all Append() calls
06:06:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.713434 1943 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648: No bootstrap required, opened a new log
06:06:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.754079 1943 raft_consensus.cc:348] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.755105 1943 raft_consensus.cc:374] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:06:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.755406 1943 raft_consensus.cc:729] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 674cc974e7ad43d1b92066aa374f8648, State: Initialized, Role: FOLLOWER
06:06:24.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.756850 1943 consensus_queue.cc:260] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.761775 1953 sys_catalog.cc:455] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } } }
06:06:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.762825 1953 sys_catalog.cc:458] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [sys.catalog]: This master's current role is: FOLLOWER
06:06:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.763651 1943 sys_catalog.cc:564] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [sys.catalog]: configured and running, proceeding with master startup.
06:06:24.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:24.793466 1887 tablet.cc:2369] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:06:24.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:24.848136 1964 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:24.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:24.848556 1964 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:24.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.833329 1848 master_runner.cc:186] Error getting master registration for 127.1.163.190:44249: OK, instance_id { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" instance_seqno: 1786428379642967 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:06:24.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:24.879535 1848 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:06:25.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:25.075815 1953 raft_consensus.cc:482] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:06:25.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:25.076413 1953 raft_consensus.cc:504] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:25.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:25.080835 1953 leader_election.cc:288] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9f9babe013fa4875adc71cf23633e9d1 (127.1.163.190:44249), d5ed02a9beb442d7b909c6236c52a184 (127.1.163.189:33127)
06:06:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:25.856369 1964 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:25.856910 1964 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.568334 1768 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 8
06:06:26.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.575829 1846 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } attempt: 8
06:06:26.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.628034 1768 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1: Bootstrap starting.
06:06:26.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.646427 1768 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1: Neither blocks nor log segments found. Creating new log.
06:06:26.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.652668 1768 log.cc:824] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1: Log is configured to *not* fsync() on all Append() calls
06:06:26.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.654763 1846 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184: Bootstrap starting.
06:06:26.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.668061 1768 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1: No bootstrap required, opened a new log
06:06:26.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.673060 1846 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184: Neither blocks nor log segments found. Creating new log.
06:06:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.680887 1846 log.cc:824] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184: Log is configured to *not* fsync() on all Append() calls
06:06:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.693364 1846 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184: No bootstrap required, opened a new log
06:06:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.743271 1768 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:26.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.745307 1768 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:06:26.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.746251 1768 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9f9babe013fa4875adc71cf23633e9d1, State: Initialized, Role: FOLLOWER
06:06:26.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.748543 1768 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.760146 1846 raft_consensus.cc:348] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.762181 1846 raft_consensus.cc:374] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:06:26.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.763108 1846 raft_consensus.cc:729] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d5ed02a9beb442d7b909c6236c52a184, State: Initialized, Role: FOLLOWER
06:06:26.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.765354 1846 consensus_queue.cc:260] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:26.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.799212 1768 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [sys.catalog]: configured and running, proceeding with master startup.
06:06:26.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.799845 1740 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "674cc974e7ad43d1b92066aa374f8648" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9f9babe013fa4875adc71cf23633e9d1" is_pre_election: true
06:06:26.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.803642 1740 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 674cc974e7ad43d1b92066aa374f8648 in term 0.
06:06:26.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.793076 1974 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } } }
06:06:26.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.806161 1974 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [sys.catalog]: This master's current role is: FOLLOWER
06:06:26.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:26.805166 1740 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54740 (request call id 2) took 1723 ms (client timeout 1750 ms). Trace:
06:06:26.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:26.807427 1740 rpcz_store.cc:269] 0811 06:06:25.081275 (+ 0us) service_pool.cc:170] Inserting onto call queue
06:06:26.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 06:06:25.081469 (+ 194us) service_pool.cc:227] Handling call
06:06:26.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 06:06:26.805048 (+1723579us) inbound_call.cc:173] Queueing success response
06:06:26.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
06:06:26.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.810406 1975 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } } }
06:06:26.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.813683 1975 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [sys.catalog]: This master's current role is: FOLLOWER
06:06:26.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.814217 1885 leader_election.cc:302] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 674cc974e7ad43d1b92066aa374f8648, 9f9babe013fa4875adc71cf23633e9d1; no voters:
06:06:26.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.819386 1975 raft_consensus.cc:482] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:06:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.824213 1846 sys_catalog.cc:564] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [sys.catalog]: configured and running, proceeding with master startup.
06:06:26.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.824995 1820 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "674cc974e7ad43d1b92066aa374f8648" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5ed02a9beb442d7b909c6236c52a184" is_pre_election: true
06:06:26.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.828682 1977 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:06:26.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.827054 1975 raft_consensus.cc:504] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.829936 1977 raft_consensus.cc:482] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:06:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.830320 1977 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 0 FOLLOWER]: Advancing to term 1
06:06:26.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:26.832990 1882 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.1.163.189:33127 timed out after 1.750s (SENT)
06:06:26.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.831485 1820 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 674cc974e7ad43d1b92066aa374f8648 in term 0.
06:06:26.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.840006 1975 leader_election.cc:288] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9f9babe013fa4875adc71cf23633e9d1 (127.1.163.190:44249), 674cc974e7ad43d1b92066aa374f8648 (127.1.163.188:35501)
06:06:26.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.842758 1740 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d5ed02a9beb442d7b909c6236c52a184" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9f9babe013fa4875adc71cf23633e9d1" is_pre_election: true
06:06:26.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:26.843600 1798 tablet.cc:2369] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:06:26.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.843657 1740 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d5ed02a9beb442d7b909c6236c52a184 in term 0.
06:06:26.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.844260 1913 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d5ed02a9beb442d7b909c6236c52a184" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "674cc974e7ad43d1b92066aa374f8648" is_pre_election: true
06:06:26.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.846375 1796 leader_election.cc:302] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9f9babe013fa4875adc71cf23633e9d1, d5ed02a9beb442d7b909c6236c52a184; no voters:
06:06:26.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.847731 1975 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:06:26.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.848130 1975 raft_consensus.cc:482] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:06:26.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.848551 1975 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 0 FOLLOWER]: Advancing to term 1
06:06:26.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.853950 1977 raft_consensus.cc:504] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:26.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.859231 1819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "674cc974e7ad43d1b92066aa374f8648" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5ed02a9beb442d7b909c6236c52a184" is_pre_election: true
06:06:26.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:26.861271 1820 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:56568 (request call id 1) took 1779 ms (client timeout 1750 ms). Trace:
06:06:26.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:26.862103 1820 rpcz_store.cc:269] 0811 06:06:25.081922 (+ 0us) service_pool.cc:170] Inserting onto call queue
06:06:26.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 06:06:25.082158 (+ 236us) service_pool.cc:227] Handling call
06:06:26.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 06:06:26.860502 (+1778344us) spinlock_profiling.cc:243] Waited 3.04 ms on lock 0x7b54000118d4. stack: 00007f4b858e2033 00007f4b858e1f30 00007f4b85463534 00000000004db2b4 00000000004db249 00007f4b8c7ee471 00007f4b8c86d2ae 00007f4b8d845c27 00007f4b8c01dc04 00007f4b8c01db92 00007f4b8c01dac1 00007f4b8c01da3c 00007f4b8c01ccf2 00007f4b8a61894c 00007f4b8a617d86 00007f4b8a61772f
06:06:26.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 06:06:26.861149 (+ 647us) inbound_call.cc:173] Queueing success response
06:06:26.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":6998400}
06:06:26.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.867832 1913 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d5ed02a9beb442d7b909c6236c52a184 in current term 1: Already voted for candidate 674cc974e7ad43d1b92066aa374f8648 in this term.
06:06:26.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.875270 1975 raft_consensus.cc:504] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:26.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.889441 1740 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "674cc974e7ad43d1b92066aa374f8648" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9f9babe013fa4875adc71cf23633e9d1"
06:06:26.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.890559 1740 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [term 0 FOLLOWER]: Advancing to term 1
06:06:26.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.896004 1820 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "674cc974e7ad43d1b92066aa374f8648" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5ed02a9beb442d7b909c6236c52a184"
06:06:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:26.897207 1964 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.897727 1820 raft_consensus.cc:2379] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 674cc974e7ad43d1b92066aa374f8648 in current term 1: Already voted for candidate d5ed02a9beb442d7b909c6236c52a184 in this term.
06:06:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:26.897732 1964 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.898325 1977 leader_election.cc:288] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [CANDIDATE]: Term 1 election: Requested vote from peers 9f9babe013fa4875adc71cf23633e9d1 (127.1.163.190:44249), d5ed02a9beb442d7b909c6236c52a184 (127.1.163.189:33127)
06:06:26.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.899058 1739 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d5ed02a9beb442d7b909c6236c52a184" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9f9babe013fa4875adc71cf23633e9d1"
06:06:26.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.900894 1913 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d5ed02a9beb442d7b909c6236c52a184" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "674cc974e7ad43d1b92066aa374f8648"
06:06:26.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.902227 1913 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate d5ed02a9beb442d7b909c6236c52a184 in current term 1: Already voted for candidate 674cc974e7ad43d1b92066aa374f8648 in this term.
06:06:26.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.904166 1796 leader_election.cc:302] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d5ed02a9beb442d7b909c6236c52a184; no voters: 674cc974e7ad43d1b92066aa374f8648, 9f9babe013fa4875adc71cf23633e9d1
06:06:26.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.904645 1975 leader_election.cc:288] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [CANDIDATE]: Term 1 election: Requested vote from peers 9f9babe013fa4875adc71cf23633e9d1 (127.1.163.190:44249), 674cc974e7ad43d1b92066aa374f8648 (127.1.163.188:35501)
06:06:26.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.907977 1975 raft_consensus.cc:2726] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
06:06:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.918975 1740 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 674cc974e7ad43d1b92066aa374f8648 in term 1.
06:06:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.926823 1885 leader_election.cc:302] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 674cc974e7ad43d1b92066aa374f8648, 9f9babe013fa4875adc71cf23633e9d1; no voters: d5ed02a9beb442d7b909c6236c52a184
06:06:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.928769 1977 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 1 FOLLOWER]: Leader election won for term 1
06:06:26.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.950405 1977 raft_consensus.cc:686] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [term 1 LEADER]: Becoming Leader. State: Replica: 674cc974e7ad43d1b92066aa374f8648, State: Running, Role: LEADER
06:06:26.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.954223 1977 consensus_queue.cc:237] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } }
06:06:26.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.972385 1988 sys_catalog.cc:455] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 674cc974e7ad43d1b92066aa374f8648. Latest consensus state: current_term: 1 leader_uuid: "674cc974e7ad43d1b92066aa374f8648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } } }
06:06:26.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:26.973129 1715 tablet.cc:2369] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:06:26.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.973857 1988 sys_catalog.cc:458] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [sys.catalog]: This master's current role is: LEADER
06:06:26.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.981508 1996 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:06:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:26.998140 1996 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:06:27.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.029217 1820 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [term 1 FOLLOWER]: Refusing update from remote peer 674cc974e7ad43d1b92066aa374f8648: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:06:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.031987 1988 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:06:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.053126 1740 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [term 1 FOLLOWER]: Refusing update from remote peer 674cc974e7ad43d1b92066aa374f8648: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:06:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.055763 1988 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
06:06:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.081260 2005 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
06:06:27.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.082386 1983 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 674cc974e7ad43d1b92066aa374f8648. Latest consensus state: current_term: 1 leader_uuid: "674cc974e7ad43d1b92066aa374f8648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } } }
06:06:27.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.086714 1983 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [sys.catalog]: This master's current role is: FOLLOWER
06:06:27.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.087728 2001 mvcc.cc:205] Tried to move back new op lower bound from 7317210673239162880 to 7317210672982048768. Current Snapshot: MvccSnapshot[applied={T|T < 7317210673239162880}]
06:06:27.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.099172 2009 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
06:06:27.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.103924 1996 catalog_manager.cc:1347] Generated new cluster ID: 99efc36c9adb40d3afc2cd229ea2cab2
06:06:27.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.103441 1977 sys_catalog.cc:455] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "674cc974e7ad43d1b92066aa374f8648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } } }
06:06:27.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.103668 2004 mvcc.cc:205] Tried to move back new op lower bound from 7317210673239162880 to 7317210672982048768. Current Snapshot: MvccSnapshot[applied={T|T < 7317210673239162880}]
06:06:27.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.104242 1996 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:06:27.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.104404 1977 sys_catalog.cc:458] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [sys.catalog]: This master's current role is: LEADER
06:06:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.120955 1983 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "674cc974e7ad43d1b92066aa374f8648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } } }
06:06:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.122031 1983 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184 [sys.catalog]: This master's current role is: FOLLOWER
06:06:27.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.131132 1974 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 674cc974e7ad43d1b92066aa374f8648. Latest consensus state: current_term: 1 leader_uuid: "674cc974e7ad43d1b92066aa374f8648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } } }
06:06:27.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.132222 1974 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [sys.catalog]: This master's current role is: FOLLOWER
06:06:27.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.155736 2007 mvcc.cc:205] Tried to move back new op lower bound from 7317210673239162880 to 7317210672982048768. Current Snapshot: MvccSnapshot[applied={T|T < 7317210673239162880}]
06:06:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.164285 1988 sys_catalog.cc:455] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "674cc974e7ad43d1b92066aa374f8648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } } }
06:06:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.165323 1988 sys_catalog.cc:458] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648 [sys.catalog]: This master's current role is: LEADER
06:06:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.172282 1974 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "674cc974e7ad43d1b92066aa374f8648" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f9babe013fa4875adc71cf23633e9d1" member_type: VOTER last_known_addr { host: "127.1.163.190" port: 44249 } } peers { permanent_uuid: "d5ed02a9beb442d7b909c6236c52a184" member_type: VOTER last_known_addr { host: "127.1.163.189" port: 33127 } } peers { permanent_uuid: "674cc974e7ad43d1b92066aa374f8648" member_type: VOTER last_known_addr { host: "127.1.163.188" port: 35501 } } }
06:06:27.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.173272 1974 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1 [sys.catalog]: This master's current role is: FOLLOWER
06:06:27.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.182757 1996 catalog_manager.cc:1370] Generated new certificate authority record
06:06:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.185514 1996 catalog_manager.cc:1504] Loading token signing keys...
06:06:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.223459 1996 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 674cc974e7ad43d1b92066aa374f8648: Generated new TSK 0
06:06:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.225569 1996 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:06:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.305104 1678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/wal
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/data
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/logs
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/data/info.pb
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.129:0
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.163.129
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.129
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:27.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:27.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:27.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:06:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:27.818080 2018 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:27.818667 2018 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:27.819353 2018 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:27.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:27.851586 2018 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:27.852636 2018 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.163.129
06:06:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.887017 2018 tablet_server_runner.cc:78] Tablet server non-default flags:
06:06:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/data
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/wal
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.129:0
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/data/info.pb
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.129
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.163.129
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/logs
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:06:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.888648 2018 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:27.890515 2018 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:27.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:27.911110 2027 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:28.090277 2005 catalog_manager.cc:1259] Loaded cluster ID: 99efc36c9adb40d3afc2cd229ea2cab2
06:06:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:28.090674 2005 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1: loading cluster ID for follower catalog manager: success
06:06:28.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:28.098767 2005 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1: acquiring CA information for follower catalog manager: success
06:06:28.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:28.104724 2005 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 9f9babe013fa4875adc71cf23633e9d1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:06:28.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:28.111951 2009 catalog_manager.cc:1259] Loaded cluster ID: 99efc36c9adb40d3afc2cd229ea2cab2
06:06:28.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:28.112681 2009 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184: loading cluster ID for follower catalog manager: success
06:06:28.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:28.124927 2009 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184: acquiring CA information for follower catalog manager: success
06:06:28.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:28.134003 2009 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d5ed02a9beb442d7b909c6236c52a184: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:06:29.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:29.505740 2028 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:06:29.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:27.912385 2030 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:29.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.505961 2018 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:29.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:27.911399 2026 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:29.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.514122 2018 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.519636 2018 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.521284 2018 hybrid_clock.cc:648] HybridClock initialized: now 1786428389521236 us; error 58 us; skew 500 ppm
06:06:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.522507 2018 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.560025 2018 webserver.cc:460] Webserver started at http://127.1.163.129:33559/ using document root <none> and password file <none>
06:06:29.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.561882 2018 fs_manager.cc:359] Metadata directory not provided
06:06:29.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.562229 2018 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:29.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.564181 2018 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:06:29.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.575081 2018 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/data/instance:
06:06:29.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06991390eb454eacbc52ff23e59484d1"
06:06:29.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:29 on dist-test-slave-pgkr"
06:06:29.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.577109 2018 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/wal/instance:
06:06:29.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06991390eb454eacbc52ff23e59484d1"
06:06:29.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:29 on dist-test-slave-pgkr"
06:06:29.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.592581 2018 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.005s sys 0.009s
06:06:29.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.606734 2042 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:29.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.608685 2018 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.006s sys 0.005s
06:06:29.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.609094 2018 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/wal
06:06:29.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06991390eb454eacbc52ff23e59484d1"
06:06:29.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:29 on dist-test-slave-pgkr"
06:06:29.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.609678 2018 fs_report.cc:383] FS layout report
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/wal
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/wal
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/data/data
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.681177 2018 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:29.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.684422 2018 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:29.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.685326 2018 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:29.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.694860 2018 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:06:29.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.705833 2018 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:06:29.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.706156 2018 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:06:29.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.706458 2018 ts_tablet_manager.cc:610] Registered 0 tablets
06:06:29.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:29.706703 2018 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.002s sys 0.000s
06:06:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.342079 2018 rpc_server.cc:307] RPC server started. Bound to: 127.1.163.129:39485
06:06:30.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.342198 2164 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.163.129:39485 every 8 connection(s)
06:06:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.346879 2018 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/data/info.pb
06:06:30.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.356117 1678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 2018
06:06:30.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.356594 1678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-0/wal/instance
06:06:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.395144 1678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/wal
06:06:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/data
06:06:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/logs
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/data/info.pb
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.130:0
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.163.130
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.130
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:06:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.402585 2165 heartbeater.cc:347] Connected to a master server at 127.1.163.188:35501
06:06:30.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.407068 2165 heartbeater.cc:464] Registering TS with master...
06:06:30.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.409761 2165 heartbeater.cc:511] Master 127.1.163.188:35501 requested a full tablet report, sending...
06:06:30.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.415773 1900 ts_manager.cc:194] Registered new tserver with Master: 06991390eb454eacbc52ff23e59484d1 (127.1.163.129:39485)
06:06:30.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.421120 1900 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.163.129:53833
06:06:30.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.424415 2169 heartbeater.cc:347] Connected to a master server at 127.1.163.190:44249
06:06:30.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.424901 2169 heartbeater.cc:464] Registering TS with master...
06:06:30.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.426462 2169 heartbeater.cc:511] Master 127.1.163.190:44249 requested a full tablet report, sending...
06:06:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.433035 1726 ts_manager.cc:194] Registered new tserver with Master: 06991390eb454eacbc52ff23e59484d1 (127.1.163.129:39485)
06:06:30.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.459266 2166 heartbeater.cc:347] Connected to a master server at 127.1.163.189:33127
06:06:30.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.460029 2166 heartbeater.cc:464] Registering TS with master...
06:06:30.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.461701 2166 heartbeater.cc:511] Master 127.1.163.189:33127 requested a full tablet report, sending...
06:06:30.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:30.467314 1809 ts_manager.cc:194] Registered new tserver with Master: 06991390eb454eacbc52ff23e59484d1 (127.1.163.129:39485)
06:06:31.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:31.129644 2176 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:31.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:31.130419 2176 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:31.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:31.131095 2176 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:31.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:31.164320 2176 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:31.165572 2176 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.163.130
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:31.201675 2176 tablet_server_runner.cc:78] Tablet server non-default flags:
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/data
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/wal
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.130:0
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/data/info.pb
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.130
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.163.130
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/logs
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:31.203398 2176 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:31.205024 2176 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:31.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:31.258988 2185 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:31.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:31.437500 2165 heartbeater.cc:503] Master 127.1.163.188:35501 was elected leader, sending a full tablet report...
06:06:33.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:33.165774 2176 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.914s user 0.659s sys 1.153s
06:06:33.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:32.629870 2183 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 2176
06:06:33.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:33.167199 2176 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.916s user 0.659s sys 1.153s
06:06:33.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:31.264464 2184 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:33.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:33.168365 2188 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1903 milliseconds
06:06:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:33.169642 2189 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.169612 2176 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:33.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.173483 2176 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.176158 2176 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:33.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.177546 2176 hybrid_clock.cc:648] HybridClock initialized: now 1786428393177512 us; error 33 us; skew 500 ppm
06:06:33.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.178365 2176 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.185097 2176 webserver.cc:460] Webserver started at http://127.1.163.130:39905/ using document root <none> and password file <none>
06:06:33.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.186048 2176 fs_manager.cc:359] Metadata directory not provided
06:06:33.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.186306 2176 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:33.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.186856 2176 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:06:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.191939 2176 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/data/instance:
06:06:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7aaa431d31684267bbcae3adb70320a5"
06:06:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:33 on dist-test-slave-pgkr"
06:06:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.193024 2176 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/wal/instance:
06:06:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7aaa431d31684267bbcae3adb70320a5"
06:06:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:33 on dist-test-slave-pgkr"
06:06:33.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.202450 2176 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.005s
06:06:33.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.208348 2201 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:33.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.209408 2176 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.209719 2176 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/wal
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7aaa431d31684267bbcae3adb70320a5"
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:33 on dist-test-slave-pgkr"
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.210011 2176 fs_report.cc:383] FS layout report
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/wal
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/wal
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/data/data
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:33.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:33.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:33.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.257017 2176 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:33.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.258450 2176 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.258957 2176 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:33.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.261492 2176 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:06:33.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.265836 2176 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:06:33.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.266021 2176 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:06:33.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.266203 2176 ts_tablet_manager.cc:610] Registered 0 tablets
06:06:33.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.266331 2176 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:06:33.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.628727 2176 rpc_server.cc:307] RPC server started. Bound to: 127.1.163.130:38277
06:06:33.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.628948 2328 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.163.130:38277 every 8 connection(s)
06:06:33.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.633230 2176 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/data/info.pb
06:06:33.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.639556 1678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 2176
06:06:33.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.640192 1678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-1/wal/instance
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.659809 1678 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/wal
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/data
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/logs
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/data/info.pb
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.131:0
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.163.131
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.131
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:06:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.714447 2329 heartbeater.cc:347] Connected to a master server at 127.1.163.188:35501
06:06:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.714991 2329 heartbeater.cc:464] Registering TS with master...
06:06:33.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.716238 2329 heartbeater.cc:511] Master 127.1.163.188:35501 requested a full tablet report, sending...
06:06:33.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.719691 1900 ts_manager.cc:194] Registered new tserver with Master: 7aaa431d31684267bbcae3adb70320a5 (127.1.163.130:38277)
06:06:33.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.720134 2334 heartbeater.cc:347] Connected to a master server at 127.1.163.190:44249
06:06:33.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.720501 2334 heartbeater.cc:464] Registering TS with master...
06:06:33.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.721577 2334 heartbeater.cc:511] Master 127.1.163.190:44249 requested a full tablet report, sending...
06:06:33.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.723033 1900 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.163.130:59695
06:06:33.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.724942 1726 ts_manager.cc:194] Registered new tserver with Master: 7aaa431d31684267bbcae3adb70320a5 (127.1.163.130:38277)
06:06:33.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.727070 2331 heartbeater.cc:347] Connected to a master server at 127.1.163.189:33127
06:06:33.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.727501 2331 heartbeater.cc:464] Registering TS with master...
06:06:33.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.729650 2331 heartbeater.cc:511] Master 127.1.163.189:33127 requested a full tablet report, sending...
06:06:33.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:33.733300 1809 ts_manager.cc:194] Registered new tserver with Master: 7aaa431d31684267bbcae3adb70320a5 (127.1.163.130:38277)
06:06:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:34.176745 2341 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:34.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:34.177480 2341 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:34.178077 2341 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:34.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:34.212117 2341 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:34.213236 2341 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.163.131
06:06:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:34.251348 2341 tablet_server_runner.cc:78] Tablet server non-default flags:
06:06:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.163.148:39549
06:06:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/data
06:06:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/wal
06:06:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.163.131:0
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/data/info.pb
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.163.131
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.163.190:44249,127.1.163.189:33127,127.1.163.188:35501
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.163.131
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/logs
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:34.252552 2341 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:34.254101 2341 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:34.279510 2351 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:34.730870 2329 heartbeater.cc:503] Master 127.1.163.188:35501 was elected leader, sending a full tablet report...
06:06:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:34.280500 2349 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:34.281564 2348 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:35.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:35.611047 2350 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:06:35.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.611186 2341 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.614665 2341 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.617170 2341 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:35.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.618563 2341 hybrid_clock.cc:648] HybridClock initialized: now 1786428395618499 us; error 53 us; skew 500 ppm
06:06:35.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.619460 2341 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:35.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.632130 2341 webserver.cc:460] Webserver started at http://127.1.163.131:45857/ using document root <none> and password file <none>
06:06:35.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.633319 2341 fs_manager.cc:359] Metadata directory not provided
06:06:35.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.633626 2341 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.634320 2341 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:06:35.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.640018 2341 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/data/instance:
06:06:35.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67854904b46f4b638dc12c50f50d5d75"
06:06:35.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:35 on dist-test-slave-pgkr"
06:06:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.641376 2341 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/wal/instance:
06:06:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67854904b46f4b638dc12c50f50d5d75"
06:06:35.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:35 on dist-test-slave-pgkr"
06:06:35.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.651213 2341 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
06:06:35.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.659384 2358 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:35.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.660543 2341 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.000s sys 0.004s
06:06:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.660933 2341 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/wal
06:06:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67854904b46f4b638dc12c50f50d5d75"
06:06:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:35 on dist-test-slave-pgkr"
06:06:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.661346 2341 fs_report.cc:383] FS layout report
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/wal
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/wal
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/data/data
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.707484 2341 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:35.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.709841 2341 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:35.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.710711 2341 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.716277 2341 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:06:35.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.724210 2341 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:06:35.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.724475 2341 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:06:35.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.724844 2341 ts_tablet_manager.cc:610] Registered 0 tablets
06:06:35.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:35.725041 2341 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:06:36.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.293972 2341 rpc_server.cc:307] RPC server started. Bound to: 127.1.163.131:41337
06:06:36.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.294793 2480 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.163.131:41337 every 8 connection(s)
06:06:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.297648 2341 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/data/info.pb
06:06:36.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.306159 1678 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 2341
06:06:36.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.307070 1678 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6870596504207115881/ts-2/wal/instance
06:06:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.380484 2482 heartbeater.cc:347] Connected to a master server at 127.1.163.189:33127
06:06:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.382956 2482 heartbeater.cc:464] Registering TS with master...
06:06:36.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.385314 2481 heartbeater.cc:347] Connected to a master server at 127.1.163.188:35501
06:06:36.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.385509 2482 heartbeater.cc:511] Master 127.1.163.189:33127 requested a full tablet report, sending...
06:06:36.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.385861 2481 heartbeater.cc:464] Registering TS with master...
06:06:36.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.387240 2481 heartbeater.cc:511] Master 127.1.163.188:35501 requested a full tablet report, sending...
06:06:36.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.390969 1809 ts_manager.cc:194] Registered new tserver with Master: 67854904b46f4b638dc12c50f50d5d75 (127.1.163.131:41337)
06:06:36.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.391508 1900 ts_manager.cc:194] Registered new tserver with Master: 67854904b46f4b638dc12c50f50d5d75 (127.1.163.131:41337)
06:06:36.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.394400 1900 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.163.131:48331
06:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.414222 2487 heartbeater.cc:347] Connected to a master server at 127.1.163.190:44249
06:06:36.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.414820 2487 heartbeater.cc:464] Registering TS with master...
06:06:36.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.416368 2487 heartbeater.cc:511] Master 127.1.163.190:44249 requested a full tablet report, sending...
06:06:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.420292 1726 ts_manager.cc:194] Registered new tserver with Master: 67854904b46f4b638dc12c50f50d5d75 (127.1.163.131:41337)
06:06:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:36.422904 1678 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
06:06:36.445 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:06:36.447 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
06:06:36.458 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.163.190"
port: 44249
}
bound_http_address {
host: "127.1.163.190"
port: 36661
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.163.189"
port: 33127
}
bound_http_address {
host: "127.1.163.189"
port: 34803
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.163.188"
port: 35501
}
bound_http_address {
host: "127.1.163.188"
port: 42655
}
}
}
06:06:36.462 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:06:36.465 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.163.129"
port: 39485
}
bound_http_address {
host: "127.1.163.129"
port: 33559
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.163.130"
port: 38277
}
bound_http_address {
host: "127.1.163.130"
port: 39905
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.163.131"
port: 41337
}
bound_http_address {
host: "127.1.163.131"
port: 45857
}
}
}
06:06:36.465 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:06:36.696 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
06:06:36.860 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoDNYHM/test-tmp
06:06:36.861 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:06:36.861 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoDNYHM/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:06:36.865 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
06:06:37.379 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:37.380 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:06:37.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:37.383348 2492 mini_oidc.cc:152] Starting JWKS server
06:06:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:37.387933 2492 webserver.cc:460] Webserver started at http://127.0.0.1:33715/ using document root <none> and password file <none>
06:06:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:37.388444 2492 mini_oidc.cc:189] Starting OIDC Discovery server
06:06:37.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:37.391768 2492 webserver.cc:460] Webserver started at http://127.0.0.1:44751/ using document root <none> and password file <none>
06:06:37.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:37.403538 2481 heartbeater.cc:503] Master 127.1.163.188:35501 was elected leader, sending a full tablet report...
06:06:37.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
06:06:37.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/principal' for realm 'KRBTEST.COM',
06:06:37.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
06:06:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:37 dist-test-slave-pgkr krb5kdc[2506](info): setting up network...
06:06:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
06:06:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:37 dist-test-slave-pgkr krb5kdc[2506](info): set up 2 sockets
06:06:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:37 dist-test-slave-pgkr krb5kdc[2506](info): commencing operation
06:06:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
06:06:38.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:38.618592 2492 mini_kdc.cc:121] Time spent starting KDC: real 1.189s user 0.012s sys 0.029s
06:06:38.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
06:06:38.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:06:38.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
06:06:38.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
06:06:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:06:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
06:06:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
06:06:38.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:06:38.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
06:06:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:06:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/test-user.keytab.
06:06:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/test-user.keytab.
06:06:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:38 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:06:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:06:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T06:06:38Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:06:38.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T06:06:38Z Disabled control of system clock
06:06:38.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.111.62@KRBTEST.COM; defaulting to no policy
06:06:38.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:06:38.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.111.62@KRBTEST.COM" created.
06:06:38.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:06:38.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.111.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.62.keytab.
06:06:38.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.111.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.62.keytab.
06:06:38.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:38.844398 2492 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:38.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:38.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/logs
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data/info.pb
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.62:45719
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.62
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.62.keytab
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.62
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:06:39.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.162313 2524 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.163065 2524 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.163782 2524 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.196039 2524 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:06:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.196311 2524 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:06:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.196695 2524 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.197021 2524 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:06:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.197324 2524 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:06:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.197631 2524 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:06:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.197855 2524 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33715/jwks.json
06:06:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:39.234856 2524 master_runner.cc:386] Master server non-default flags:
06:06:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data
06:06:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal
06:06:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:06:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.62.keytab
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.62
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.62:45719
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data/info.pb
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.62
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/logs
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:39.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:39.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:39.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:39.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:39.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:39.236409 2524 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:39.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:39.237792 2524 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.268698 2530 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:40.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.268939 2532 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:40.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:39.268872 2534 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:40.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:40.615130 2533 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1347 milliseconds
06:06:40.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.615440 2524 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.616869 2524 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.620570 2524 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:40.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.622220 2524 hybrid_clock.cc:648] HybridClock initialized: now 1786428400622142 us; error 82 us; skew 500 ppm
06:06:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.623406 2524 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:40.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:40 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428400, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:06:40.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.637101 2524 init.cc:377] Logged in from keytab as kudu/127.2.111.62@KRBTEST.COM (short username kudu)
06:06:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.648238 2524 webserver.cc:460] Webserver started at http://127.2.111.62:32799/ using document root <none> and password file <none>
06:06:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.650046 2524 fs_manager.cc:359] Metadata directory not provided
06:06:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.650279 2524 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.651103 2524 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:06:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.657670 2524 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data/instance:
06:06:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3506a170a13d4b89b8adfd6af2db46e3"
06:06:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:40 on dist-test-slave-pgkr"
06:06:40.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.658941 2524 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal/instance:
06:06:40.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3506a170a13d4b89b8adfd6af2db46e3"
06:06:40.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:40 on dist-test-slave-pgkr"
06:06:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.669539 2524 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.009s sys 0.000s
06:06:40.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.677734 2545 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.679100 2524 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.003s sys 0.004s
06:06:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.679415 2524 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal
06:06:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3506a170a13d4b89b8adfd6af2db46e3"
06:06:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:40 on dist-test-slave-pgkr"
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.679739 2524 fs_report.cc:383] FS layout report
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data/data
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.751191 2524 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.758634 2524 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.759259 2524 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:40.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.952041 2524 rpc_server.cc:307] RPC server started. Bound to: 127.2.111.62:45719
06:06:40.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.952222 2601 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.111.62:45719 every 8 connection(s)
06:06:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.955199 2524 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data/info.pb
06:06:40.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.965005 2492 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 2524
06:06:40.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.965626 2492 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal/instance
06:06:40.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.966804 2602 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:06:40.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:40.992277 2602 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } has no permanent_uuid. Determining permanent_uuid...
06:06:40.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.111.61@KRBTEST.COM; defaulting to no policy
06:06:41.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:06:41.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.111.61@KRBTEST.COM" created.
06:06:41.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:41 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428400, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.62@KRBTEST.COM for kudu/127.2.111.62@KRBTEST.COM
06:06:41.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:06:41.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.111.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.61.keytab.
06:06:41.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.111.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.61.keytab.
06:06:41.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:41.052064 2492 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/logs
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data/info.pb
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.61:37299
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.61
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.61.keytab
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.61
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:06:41.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:41.059652 2602 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } has no permanent_uuid. Determining permanent_uuid...
06:06:41.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.064742 2547 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:41.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.074959 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.61:37299: Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:41.124768 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } attempt: 1
06:06:41.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.129271 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.61:37299: Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:41.197733 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } attempt: 2
06:06:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.202064 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.61:37299: Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:41.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:41.293529 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } attempt: 3
06:06:41.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.298432 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.61:37299: Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:41.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:41.441996 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } attempt: 4
06:06:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.451519 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.61:37299: Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:41.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.504134 2609 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:41.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.504699 2609 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:41.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.505354 2609 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:41.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.541450 2609 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:06:41.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.541745 2609 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:06:41.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.542279 2609 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:41.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.542689 2609 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:06:41.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.543028 2609 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:06:41.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.543370 2609 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:06:41.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.543740 2609 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33715/jwks.json
06:06:41.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:41.583439 2609 master_runner.cc:386] Master server non-default flags:
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.61.keytab
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.61
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.61:37299
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data/info.pb
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.61
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/logs
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:41.585098 2609 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:41.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:41.586426 2609 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:41.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.624651 2619 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:41.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:41.751103 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } attempt: 5
06:06:41.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.757131 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.61:37299: Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:42.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:42.304711 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } attempt: 6
06:06:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:42.317752 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.61:37299: Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:43.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.625708 2618 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:43.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:41.627054 2622 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:43.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.093003 2609 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:43.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:43.092856 2621 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:06:43.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.097087 2609 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:43.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.101622 2609 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:43.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.105448 2609 hybrid_clock.cc:648] HybridClock initialized: now 1786428403105345 us; error 64 us; skew 500 ppm
06:06:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.106457 2609 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:43.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:43 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428403, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:06:43.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.117300 2609 init.cc:377] Logged in from keytab as kudu/127.2.111.61@KRBTEST.COM (short username kudu)
06:06:43.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.132963 2609 webserver.cc:460] Webserver started at http://127.2.111.61:36113/ using document root <none> and password file <none>
06:06:43.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.134090 2609 fs_manager.cc:359] Metadata directory not provided
06:06:43.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.134357 2609 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.135036 2609 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:06:43.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.142575 2609 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data/instance:
06:06:43.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bb44b178f58f442a8a61ba0e03363667"
06:06:43.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:43 on dist-test-slave-pgkr"
06:06:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.144371 2609 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal/instance:
06:06:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bb44b178f58f442a8a61ba0e03363667"
06:06:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:43 on dist-test-slave-pgkr"
06:06:43.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.159550 2609 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.014s sys 0.001s
06:06:43.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.171851 2633 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.173202 2609 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.001s sys 0.008s
06:06:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.173581 2609 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bb44b178f58f442a8a61ba0e03363667"
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:43 on dist-test-slave-pgkr"
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.173938 2609 fs_report.cc:383] FS layout report
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data/data
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:43.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.233642 2609 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:43.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.243975 2609 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:43.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.245183 2609 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.378654 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } attempt: 7
06:06:43.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:43.398010 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.61:37299: Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:43.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.532789 2609 rpc_server.cc:307] RPC server started. Bound to: 127.2.111.61:37299
06:06:43.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.533062 2690 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.111.61:37299 every 8 connection(s)
06:06:43.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.536379 2609 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data/info.pb
06:06:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.541838 2492 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 2609
06:06:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.542346 2492 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal/instance
06:06:43.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.555280 2691 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:06:43.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.111.60@KRBTEST.COM; defaulting to no policy
06:06:43.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.582695 2691 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } has no permanent_uuid. Determining permanent_uuid...
06:06:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:06:43.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.111.60@KRBTEST.COM" created.
06:06:43.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:43 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428403, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.61@KRBTEST.COM for kudu/127.2.111.62@KRBTEST.COM
06:06:43.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.811795 2691 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } has no permanent_uuid. Determining permanent_uuid...
06:06:43.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:43 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428403, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.61@KRBTEST.COM for kudu/127.2.111.61@KRBTEST.COM
06:06:43.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:43.948830 2691 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } has no permanent_uuid. Determining permanent_uuid...
06:06:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:43.953191 2637 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:43.967549 2691 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:44.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:44.017402 2691 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 1
06:06:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.022925 2691 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:44.091318 2691 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 2
06:06:44.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.096413 2691 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:44.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:44.187893 2691 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 3
06:06:44.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.191886 2691 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:44.335258 2691 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 4
06:06:44.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.339781 2691 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:44.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:06:44.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.111.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.60.keytab.
06:06:44.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.111.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.60.keytab.
06:06:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.453195 2492 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.2.111.60: real 0.908s user 0.004s sys 0.032s
06:06:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:44.455058 2492 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal
06:06:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data
06:06:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/logs
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data/info.pb
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.60:35377
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.60
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.60.keytab
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.60
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:06:44.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:44.639227 2691 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 5
06:06:44.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.643693 2691 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.913839 2703 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.914593 2703 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:44.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.915323 2703 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:44.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.960603 2703 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:06:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.960844 2703 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:06:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.961201 2703 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.961481 2703 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:06:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.961688 2703 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:06:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.961875 2703 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:06:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:44.962023 2703 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33715/jwks.json
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:45.001860 2703 master_runner.cc:386] Master server non-default flags:
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.60.keytab
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.60
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.60:35377
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data/info.pb
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.60
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/logs
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:06:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:45.003255 2703 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:45.004295 2703 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:45.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:45.033074 2710 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:45.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:45.191352 2691 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 6
06:06:45.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:45.202457 2691 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:45.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:45.488940 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } attempt: 8
06:06:45.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:45 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428400, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.62@KRBTEST.COM for kudu/127.2.111.61@KRBTEST.COM
06:06:45.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:45.582021 2602 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } has no permanent_uuid. Determining permanent_uuid...
06:06:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:45.610087 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:45.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:45.675738 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 1
06:06:45.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:45.693964 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:45.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:45.747511 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 2
06:06:45.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:45.754171 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:45.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:45.830744 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 3
06:06:45.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:45.839440 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:45.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:45.995203 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 4
06:06:46.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:46.003846 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:46.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.262998 2691 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 7
06:06:46.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:46.279418 2691 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:46.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.300462 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 5
06:06:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:46.305373 2549 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111) [suppressed 12 similar messages]
06:06:46.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:46.311256 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:45.036126 2712 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:45.033653 2709 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:46.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:46.723307 2711 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1686 milliseconds
06:06:46.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.723424 2703 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:46.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.724711 2703 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:46.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.728433 2703 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:46.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.730003 2703 hybrid_clock.cc:648] HybridClock initialized: now 1786428406729951 us; error 49 us; skew 500 ppm
06:06:46.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.731237 2703 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:46 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428406, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:06:46.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.746982 2703 init.cc:377] Logged in from keytab as kudu/127.2.111.60@KRBTEST.COM (short username kudu)
06:06:46.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.756960 2703 webserver.cc:460] Webserver started at http://127.2.111.60:44337/ using document root <none> and password file <none>
06:06:46.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.757943 2703 fs_manager.cc:359] Metadata directory not provided
06:06:46.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.758189 2703 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:46.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.758795 2703 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:06:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.764801 2703 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data/instance:
06:06:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e27f24c0719a4d6a90d89d362dde86f8"
06:06:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:46 on dist-test-slave-pgkr"
06:06:46.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.766191 2703 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal/instance:
06:06:46.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e27f24c0719a4d6a90d89d362dde86f8"
06:06:46.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:46 on dist-test-slave-pgkr"
06:06:46.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.778024 2703 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.005s sys 0.005s
06:06:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.786926 2729 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.788075 2703 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.000s sys 0.007s
06:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.788447 2703 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal
06:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e27f24c0719a4d6a90d89d362dde86f8"
06:06:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:46 on dist-test-slave-pgkr"
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.788818 2703 fs_report.cc:383] FS layout report
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data/data
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:46.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:46.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:46.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:46.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.832770 2602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } attempt: 6
06:06:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.840830 2703 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:46.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:46.843031 2602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.111.60:35377: Network error: Client connection negotiation failed: client connection to 127.2.111.60:35377: connect: Connection refused (error 111)
06:06:46.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.848927 2703 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:46.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:46.849460 2703 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:47.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.092423 2703 rpc_server.cc:307] RPC server started. Bound to: 127.2.111.60:35377
06:06:47.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.092556 2788 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.111.60:35377 every 8 connection(s)
06:06:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.095767 2703 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data/info.pb
06:06:47.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.102646 2492 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 2703
06:06:47.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.103487 2492 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal/instance
06:06:47.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.108526 2789 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:06:47.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.140388 2789 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } has no permanent_uuid. Determining permanent_uuid...
06:06:47.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:47 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.111.60@KRBTEST.COM
06:06:47.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:47 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428406, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.60@KRBTEST.COM for kudu/127.2.111.62@KRBTEST.COM
06:06:47.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.208662 2789 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } has no permanent_uuid. Determining permanent_uuid...
06:06:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:47 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.111.61@KRBTEST.COM
06:06:47.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:47 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428406, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.60@KRBTEST.COM for kudu/127.2.111.61@KRBTEST.COM
06:06:47.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.258735 2789 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } has no permanent_uuid. Determining permanent_uuid...
06:06:47.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:47 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.111.62@KRBTEST.COM
06:06:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:47 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428406, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.60@KRBTEST.COM for kudu/127.2.111.60@KRBTEST.COM
06:06:47.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.294179 2492 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
06:06:47.301 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:47.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
06:06:47.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.303923 2789 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: Bootstrap starting.
06:06:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.319802 2789 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: Neither blocks nor log segments found. Creating new log.
06:06:47.323 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
06:06:47.324 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
06:06:47.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.325093 2789 log.cc:824] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: Log is configured to *not* fsync() on all Append() calls
06:06:47.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.331251 2789 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: No bootstrap required, opened a new log
06:06:47.332 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.111.62"
port: 45719
}
bound_http_address {
host: "127.2.111.62"
port: 32799
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.111.61"
port: 37299
}
bound_http_address {
host: "127.2.111.61"
port: 36113
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.111.60"
port: 35377
}
bound_http_address {
host: "127.2.111.60"
port: 44337
}
}
}
06:06:47.332 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:06:47.333 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:47.333 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
06:06:47.335 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
06:06:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.358970 2789 raft_consensus.cc:348] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:06:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.360004 2789 raft_consensus.cc:374] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:06:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.360330 2789 raft_consensus.cc:729] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e27f24c0719a4d6a90d89d362dde86f8, State: Initialized, Role: FOLLOWER
06:06:47.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.361393 2789 consensus_queue.cc:260] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:06:47.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.365011 2801 sys_catalog.cc:455] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } } }
06:06:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.366146 2801 sys_catalog.cc:458] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [sys.catalog]: This master's current role is: FOLLOWER
06:06:47.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.367057 2789 sys_catalog.cc:564] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [sys.catalog]: configured and running, proceeding with master startup.
06:06:47.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:47 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428407, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:06:47.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:06:47.397 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:47.399 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5cc
06:06:47.410 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:06:47.412 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5cc
06:06:47.415 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:06:47.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.417063 2815 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:47.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.417603 2815 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:47.422 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
06:06:47.426 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
06:06:47.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
06:06:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.396593 2703 master_runner.cc:186] Error getting master registration for 127.2.111.62:45719: OK, instance_id { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" instance_seqno: 1786428400895458 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:06:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.449512 2703 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:06:47.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:47 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428407, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:06:47.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:06:47.497 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:47.497 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5cc
06:06:47.499 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:06:47.500 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.111.61:37299
06:06:47.502 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
06:06:47.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.503108 2492 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu with pid 2609
06:06:47.535 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:47.535 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.111.60:35377
06:06:47.536 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
06:06:47.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.536644 2492 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu with pid 2703
06:06:47.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:47.572 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.111.62:45719
06:06:47.573 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
06:06:47.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.573619 2492 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu with pid 2524
06:06:47.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:47.603 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.111.61:37299
06:06:47.604 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:47.606745 2492 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/logs
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data/info.pb
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.61:37299
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.61
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36113
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.61.keytab
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.61
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:47.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:47.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:47.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:47.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:47.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:47.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:06:47.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.950462 2819 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:47.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.951248 2819 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:47.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.951989 2819 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:47.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.985603 2819 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:06:47.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.985888 2819 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:06:47.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.986297 2819 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:47.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.986682 2819 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:06:47.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.987012 2819 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:06:47.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.987329 2819 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:06:47.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:47.987564 2819 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33715/jwks.json
06:06:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:48.024685 2819 master_runner.cc:386] Master server non-default flags:
06:06:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.61.keytab
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.61
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.61:37299
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data/info.pb
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.61
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36113
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/logs
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:48.026149 2819 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:48.027446 2819 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:48.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:48.048488 2829 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:49.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:48.048719 2831 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:49.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:48.049180 2828 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:49.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.260416 2830 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:06:49.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.260532 2819 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.265201 2819 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:49.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.267764 2819 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.269116 2819 hybrid_clock.cc:648] HybridClock initialized: now 1786428409269082 us; error 39 us; skew 500 ppm
06:06:49.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.269953 2819 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:49.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:49 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428409, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:06:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.276139 2819 init.cc:377] Logged in from keytab as kudu/127.2.111.61@KRBTEST.COM (short username kudu)
06:06:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.281563 2819 webserver.cc:460] Webserver started at http://127.2.111.61:36113/ using document root <none> and password file <none>
06:06:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.282519 2819 fs_manager.cc:359] Metadata directory not provided
06:06:49.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.282819 2819 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:49.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.291692 2819 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.008s sys 0.000s
06:06:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.296216 2839 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:49.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.297170 2819 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
06:06:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.297540 2819 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal
06:06:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bb44b178f58f442a8a61ba0e03363667"
06:06:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:43 on dist-test-slave-pgkr"
06:06:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.300220 2819 fs_report.cc:383] FS layout report
06:06:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal
06:06:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal
06:06:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data/data
06:06:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:49.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.336548 2819 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.342366 2819 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.342932 2819 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:49.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.417640 2819 rpc_server.cc:307] RPC server started. Bound to: 127.2.111.61:37299
06:06:49.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.417734 2899 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.111.61:37299 every 8 connection(s)
06:06:49.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.420778 2819 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/data/info.pb
06:06:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.425519 2492 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 2819
06:06:49.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:49.426 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.111.60:35377
06:06:49.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.427951 2492 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/logs
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data/info.pb
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.60:35377
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.60
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44337
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.60.keytab
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.60
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:06:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.433640 2900 sys_catalog.cc:263] Verifying existing consensus state
06:06:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260811 06:06:49.434677 2900 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:06:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.435144 2819 master.cc:537] Master@127.2.111.61:37299 shutting down...
06:06:49.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.510546 2819 master.cc:552] Master@127.2.111.61:37299 shutdown complete.
06:06:49.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:06:49.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.761410 2902 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:49.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.761951 2902 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:49.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.762537 2902 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:49.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.793143 2902 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:06:49.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.793433 2902 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:06:49.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.793713 2902 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.793972 2902 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:06:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.794175 2902 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:06:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.794353 2902 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:06:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.794535 2902 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33715/jwks.json
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.830693 2902 master_runner.cc:386] Master server non-default flags:
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.60.keytab
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.60
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.60:35377
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data/info.pb
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.60
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44337
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/logs
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.832176 2902 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.833650 2902 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.852115 2908 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:51.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.852137 2909 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:51.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:49.854538 2902 server_base.cc:1034] running on GCE node
06:06:51.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:49.853103 2911 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:51.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.064827 2902 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:51.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.068167 2902 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.069617 2902 hybrid_clock.cc:648] HybridClock initialized: now 1786428411069544 us; error 87 us; skew 500 ppm
06:06:51.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.070479 2902 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:51.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:51 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428411, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:06:51.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.076798 2902 init.cc:377] Logged in from keytab as kudu/127.2.111.60@KRBTEST.COM (short username kudu)
06:06:51.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.082727 2902 webserver.cc:460] Webserver started at http://127.2.111.60:44337/ using document root <none> and password file <none>
06:06:51.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.083755 2902 fs_manager.cc:359] Metadata directory not provided
06:06:51.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.084026 2902 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:51.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.092324 2902 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
06:06:51.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.097386 2922 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:51.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.098582 2902 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
06:06:51.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.098917 2902 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal
06:06:51.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e27f24c0719a4d6a90d89d362dde86f8"
06:06:51.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:46 on dist-test-slave-pgkr"
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.101336 2902 fs_report.cc:383] FS layout report
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/wal
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data/data
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.154421 2902 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:51.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.160495 2902 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:51.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.161121 2902 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:51.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.237059 2902 rpc_server.cc:307] RPC server started. Bound to: 127.2.111.60:35377
06:06:51.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.237170 2977 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.111.60:35377 every 8 connection(s)
06:06:51.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.240168 2902 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-2/data/info.pb
06:06:51.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.247334 2492 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 2902
06:06:51.249 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:51.249 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.111.62:45719
06:06:51.250 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
06:06:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.251809 2492 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu
06:06:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal
06:06:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data
06:06:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/logs
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data/info.pb
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.62:45719
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.62
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32799
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.62.keytab
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.62
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:06:51.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.253772 2978 sys_catalog.cc:263] Verifying existing consensus state
06:06:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.260051 2978 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: Bootstrap starting.
06:06:51.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.298440 2978 log.cc:824] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: Log is configured to *not* fsync() on all Append() calls
06:06:51.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.303344 2978 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:06:51.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.304234 2978 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: Bootstrap complete.
06:06:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.327580 2978 raft_consensus.cc:348] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:06:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.330312 2978 raft_consensus.cc:374] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:06:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.330646 2978 raft_consensus.cc:729] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e27f24c0719a4d6a90d89d362dde86f8, State: Initialized, Role: FOLLOWER
06:06:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.331524 2978 consensus_queue.cc:260] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:06:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.334554 2983 sys_catalog.cc:455] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } } }
06:06:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.336951 2983 sys_catalog.cc:458] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [sys.catalog]: This master's current role is: FOLLOWER
06:06:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.339532 2978 sys_catalog.cc:564] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [sys.catalog]: configured and running, proceeding with master startup.
06:06:51.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.370411 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:51.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.371148 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:51.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.367897 2902 master_runner.cc:186] Error getting master registration for 127.2.111.62:45719: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111),
06:06:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.396885 2902 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:06:51.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.416865 2929 tablet.cc:2369] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:06:51.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.596536 2980 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:06:51.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.597086 2980 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:06:51.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.597683 2980 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:06:51.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.632093 2980 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:06:51.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.632392 2980 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:06:51.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.632882 2980 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:06:51.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.633239 2980 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:06:51.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.633590 2980 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:06:51.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.633941 2980 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:06:51.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.634222 2980 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33715/jwks.json
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.672658 2980 master_runner.cc:386] Master server non-default flags:
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.111.20:41707
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.111.62:45719,127.2.111.61:37299,127.2.111.60:35377
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/krb5kdc/kudu_127.2.111.62.keytab
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.111.62
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.111.62:45719
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33715/jwks.json
06:06:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:06:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:06:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:06:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:06:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:06:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data/info.pb
06:06:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.111.62
06:06:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32799
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/logs
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 05:18:43 UTC on ccycloud.quasar-tzddmv.root.comops.site
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4579726
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.674209 2980 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:06:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.675602 2980 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:06:51.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.685737 2983 raft_consensus.cc:482] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:06:51.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.686456 2983 raft_consensus.cc:504] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:06:51.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.694640 2927 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111)
06:06:51.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.696712 2983 leader_election.cc:288] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719), bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299)
06:06:51.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.699620 2927 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719): Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111)
06:06:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.700904 3005 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.702162 2925 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299): Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.702548 2925 leader_election.cc:302] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e27f24c0719a4d6a90d89d362dde86f8; no voters: 3506a170a13d4b89b8adfd6af2db46e3, bb44b178f58f442a8a61ba0e03363667
06:06:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:51.704311 2983 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:06:52.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:52.377190 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:52.378062 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:53.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:53.384644 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:53.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:53.385593 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.393522 3014 raft_consensus.cc:482] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:06:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.394337 3014 raft_consensus.cc:504] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:06:53.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.405346 3014 leader_election.cc:288] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719), bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299)
06:06:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:53.413316 2927 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719): Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111)
06:06:53.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:53.414463 2925 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299): Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:53.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.415036 2925 leader_election.cc:302] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e27f24c0719a4d6a90d89d362dde86f8; no voters: 3506a170a13d4b89b8adfd6af2db46e3, bb44b178f58f442a8a61ba0e03363667
06:06:53.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.416555 3014 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:06:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:53.092355 3004 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 2980
06:06:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:53.568109 3004 kernel_stack_watchdog.cc:198] Thread 2980 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 384ms:
06:06:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
06:06:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
06:06:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
06:06:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
06:06:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:51.705777 3006 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:53.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:53.570645 2980 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.865s user 0.522s sys 1.135s
06:06:53.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:53.570760 3010 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:06:53.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:53.571017 2980 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.866s user 0.522s sys 1.135s
06:06:53.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:53.575423 3009 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1863 milliseconds
06:06:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.575449 2980 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:06:53.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.577167 2980 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:06:53.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.583202 2980 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:06:53.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.584625 2980 hybrid_clock.cc:648] HybridClock initialized: now 1786428413584548 us; error 70 us; skew 500 ppm
06:06:53.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.585850 2980 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:06:53.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:53 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428413, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:06:53.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.599334 2980 init.cc:377] Logged in from keytab as kudu/127.2.111.62@KRBTEST.COM (short username kudu)
06:06:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.605715 2980 webserver.cc:460] Webserver started at http://127.2.111.62:32799/ using document root <none> and password file <none>
06:06:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.606694 2980 fs_manager.cc:359] Metadata directory not provided
06:06:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.606947 2980 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:06:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.616171 2980 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.001s
06:06:53.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.620828 3024 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:06:53.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.622325 2980 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.005s
06:06:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.622841 2980 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data,/tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal
06:06:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3506a170a13d4b89b8adfd6af2db46e3"
06:06:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 06:06:40 on dist-test-slave-pgkr"
06:06:53.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.626397 2980 fs_report.cc:383] FS layout report
06:06:53.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:06:53.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal
06:06:53.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data/data
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:06:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:06:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.680316 2980 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:06:53.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.686942 2980 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:06:53.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.687354 2980 kserver.cc:163] Server-wide thread pool size limit: 3276
06:06:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.787730 2980 rpc_server.cc:307] RPC server started. Bound to: 127.2.111.62:45719
06:06:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.787789 3077 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.111.62:45719 every 8 connection(s)
06:06:53.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.790668 2980 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/data/info.pb
06:06:53.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.793854 2492 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu as pid 2980
06:06:53.794 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:06:53.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.803848 3078 sys_catalog.cc:263] Verifying existing consensus state
06:06:53.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260811 06:06:53.804800 3078 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:06:53.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.805351 2980 master.cc:537] Master@127.2.111.62:45719 shutting down...
06:06:53.865 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:53.890776 2980 master.cc:552] Master@127.2.111.62:45719 shutdown complete.
06:06:53.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskoDNYHM/test-tmp/mini-kudu-cluster6725781401850464374/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:06:53.945 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.111.60:35377(127.2.111.60:35377)] connecting to peer
06:06:53.951 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:53.993 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.2.111.60:35377(127.2.111.60:35377)] Successfully connected to peer
06:06:53.996 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:53.996 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:53.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:53.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.000 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.000 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.076 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
06:06:54.086 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.111.60
06:06:54.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:06:54 dist-test-slave-pgkr krb5kdc[2506](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428407, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.111.60@KRBTEST.COM
06:06:54.354 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xb1856326, L:/127.0.0.1:51022 - R:/127.2.111.60:35377] using SASL/GSSAPI
06:06:54.375 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:54.381 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:54.385 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:54.388 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:54.389636 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:54.389906 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:54.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.400 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:54.421 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:54.423 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:54.425 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.427 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.427 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.431 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:54.440 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:54.441 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:54.443 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.444 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.444 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.444 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.448 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:54.481 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:54.482 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:54.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.486 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.487 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.487 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.487 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.490 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:54.520 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:54.522 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:54.524 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.524 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.527 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.527 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.528 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.529 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:54.581 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:54.583 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:54.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.586 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.587 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.587 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.587 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.589 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:54.721 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:54.723 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:54.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.726 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.727 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.727 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.731 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:54.981 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:54.983 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:54.984 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:54.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:54.992 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:55.120 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:55.122 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:55.124 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:55.124 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:55.124 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:55.124 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:55.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:55.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:55.126 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:55.381 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:55.383 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:55.387 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:55.387 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:55.387 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:55.388 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:55.388 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:55.388 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:55.389 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:55.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:55.392791 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:55.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:55.393074 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:55.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:55.444370 3085 raft_consensus.cc:482] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:06:55.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:55.444813 3085 raft_consensus.cc:504] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:06:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:55.446273 3085 leader_election.cc:288] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719), bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299)
06:06:55.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:55.452716 2927 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719): Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111)
06:06:55.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:55.453157 2925 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299): Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:55.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:55.453528 2925 leader_election.cc:302] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e27f24c0719a4d6a90d89d362dde86f8; no voters: 3506a170a13d4b89b8adfd6af2db46e3, bb44b178f58f442a8a61ba0e03363667
06:06:55.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:55.454150 3085 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:06:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:56.396166 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:56.396471 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:56.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:56.399359 2902 master_runner.cc:186] Error getting master registration for 127.2.111.62:45719: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111),
06:06:56.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:56.399689 2902 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:06:56.761 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:56.762 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:56.764 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:56.764 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:56.764 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:56.764 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:56.764 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:56.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:56.766 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:57.399185 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:57.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:57.399449 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:58.021 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:58.023 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:58.024 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:58.024 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:58.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:58.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:58.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:58.026 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:58.027 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:58.402925 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:58.403328 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:59.341 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:06:59.343 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:06:59.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:59.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:59.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:59.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:59.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:06:59.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:06:59.347 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:06:59.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:59.405974 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:06:59.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:59.406248 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:06:59.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:59.711606 3094 raft_consensus.cc:482] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:06:59.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:59.711979 3094 raft_consensus.cc:504] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:06:59.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:59.713400 3094 leader_election.cc:288] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719), bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299)
06:06:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:59.715476 2927 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111) [suppressed 5 similar messages]
06:06:59.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:59.718765 2925 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299): Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:06:59.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:06:59.719178 2927 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719): Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111)
06:06:59.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:59.719623 2927 leader_election.cc:302] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e27f24c0719a4d6a90d89d362dde86f8; no voters: 3506a170a13d4b89b8adfd6af2db46e3, bb44b178f58f442a8a61ba0e03363667
06:06:59.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:06:59.720268 3094 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:07:00.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:00.331202 1939 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2f20 after lost signal to thread 1856
06:07:00.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:00.331910 1939 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd6e0 after lost signal to thread 1942
06:07:00.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:00.409833 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:00.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:00.410212 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:01.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:07:01 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428421, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:07:01.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:01.081190 2917 init.cc:295] Successfully reacquired a new kerberos TGT
06:07:01.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:01.403641 2902 master_runner.cc:186] Error getting master registration for 127.2.111.62:45719: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111),
06:07:01.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:01.404278 2902 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:07:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:01.413018 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:01.413307 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:02.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:02.416560 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:02.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:02.416893 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:02.541 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:07:02.544 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:07:02.544 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:02.544 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:02.544 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:02.546 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:02.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:02.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:02.547 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:07:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:02.832197 1763 debug-util.cc:398] Leaking SignalData structure 0x7b080009f2e0 after lost signal to thread 1692
06:07:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:02.832775 1763 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5f20 after lost signal to thread 1767
06:07:03.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:03.419996 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:03.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:03.420276 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:04.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:04.423480 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:04.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:04.423774 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:04.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:04.572913 3105 raft_consensus.cc:482] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:07:04.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:04.573365 3105 raft_consensus.cc:504] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:07:04.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:04.575031 3105 leader_election.cc:288] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719), bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299)
06:07:04.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:04.579762 2927 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719): Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111)
06:07:04.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:04.580343 2925 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299): Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:07:04.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:04.580693 2925 leader_election.cc:302] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e27f24c0719a4d6a90d89d362dde86f8; no voters: 3506a170a13d4b89b8adfd6af2db46e3, bb44b178f58f442a8a61ba0e03363667
06:07:04.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:04.581326 3105 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:07:04.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:04.627769 2974 debug-util.cc:398] Leaking SignalData structure 0x7b080007a2e0 after lost signal to thread 2903
06:07:04.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:04.628381 2974 debug-util.cc:398] Leaking SignalData structure 0x7b08000b95c0 after lost signal to thread 2977
06:07:05.361 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:07:05.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:05.364 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:05.364 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:05.364 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:07:05.366 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:05.366 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:05.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:05.367 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:07:05.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:05.428068 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:05.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:05.428418 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:06.431381 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:06.431654 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:06.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:06.462529 2902 master_runner.cc:186] Error getting master registration for 127.2.111.62:45719: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111),
06:07:06.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:06.462961 2902 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:07:06.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:06.736636 3111 raft_consensus.cc:482] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:07:06.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:06.737030 3111 raft_consensus.cc:504] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:07:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:06.739163 3111 leader_election.cc:288] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719), bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299)
06:07:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:06.739964 2927 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111) [suppressed 3 similar messages]
06:07:06.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:06.743106 2927 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719): Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111)
06:07:06.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:06.743713 2925 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299): Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:07:06.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:06.743999 2925 leader_election.cc:302] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e27f24c0719a4d6a90d89d362dde86f8; no voters: 3506a170a13d4b89b8adfd6af2db46e3, bb44b178f58f442a8a61ba0e03363667
06:07:06.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:06.744520 3111 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:07:07.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:07.434897 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:07.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:07.435160 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:08.241 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:07:08.243 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:07:08.244 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:08.244 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:08.244 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:08.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:08.245 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:08.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:08.247 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:07:08.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:08.438305 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:08.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:08.438578 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:09.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:09.441759 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:09.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:09.441996 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:09.560 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:07:09.562 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:09.563 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:07:09.563 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:09.563 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:09.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:09.565 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:09.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:09.569 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:07:10.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:10.446146 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:10.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:10.446468 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:11.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:07:11 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428431, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:07:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:11.083876 2917 init.cc:295] Successfully reacquired a new kerberos TGT
06:07:11.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:11.449445 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:11.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:11.449723 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:11.466212 2902 master_runner.cc:186] Error getting master registration for 127.2.111.62:45719: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111),
06:07:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:11.466576 2902 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:07:11.681 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:07:11.683 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:07:11.684 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:11.684 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:11.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:11.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:11.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:11.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:11.687 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:07:12.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:12.452592 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:12.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:12.452834 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:13.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:13.455895 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:13.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:13.456143 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:14.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:14.459118 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:14.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:14.459369 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:15.462320 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:15.462568 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:15.481 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:07:15.483 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:07:15.484 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:15.484 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:15.484 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:15.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:15.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:15.485 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:15.486 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:07:16.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:16.465905 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:16.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:16.466161 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:16.468976 2902 master_runner.cc:186] Error getting master registration for 127.2.111.62:45719: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111),
06:07:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:16.469309 2902 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:07:16.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:16.611462 3132 raft_consensus.cc:482] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:07:16.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:16.612030 3132 raft_consensus.cc:504] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3506a170a13d4b89b8adfd6af2db46e3" member_type: VOTER last_known_addr { host: "127.2.111.62" port: 45719 } } peers { permanent_uuid: "bb44b178f58f442a8a61ba0e03363667" member_type: VOTER last_known_addr { host: "127.2.111.61" port: 37299 } } peers { permanent_uuid: "e27f24c0719a4d6a90d89d362dde86f8" member_type: VOTER last_known_addr { host: "127.2.111.60" port: 35377 } }
06:07:16.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:16.613909 3132 leader_election.cc:288] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719), bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299)
06:07:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:16.615319 2927 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111) [suppressed 1 similar messages]
06:07:16.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:16.617828 2927 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3506a170a13d4b89b8adfd6af2db46e3 (127.2.111.62:45719): Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111)
06:07:16.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:16.618430 2925 leader_election.cc:334] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb44b178f58f442a8a61ba0e03363667 (127.2.111.61:37299): Network error: Client connection negotiation failed: client connection to 127.2.111.61:37299: connect: Connection refused (error 111)
06:07:16.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:16.618779 2925 leader_election.cc:302] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e27f24c0719a4d6a90d89d362dde86f8; no voters: 3506a170a13d4b89b8adfd6af2db46e3, bb44b178f58f442a8a61ba0e03363667
06:07:16.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:16.619385 3132 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:07:17.440 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:07:17.441 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:07:17.442 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:17.443 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:17.443 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:17.443 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:17.443 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:17.444 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:17.446 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:07:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:17.469197 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:17.469445 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:18.472361 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:18.472615 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:18.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:18.559119 2161 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8180 after lost signal to thread 2020
06:07:18.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:18.560325 2161 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8d40 after lost signal to thread 2164
06:07:19.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:19.476883 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:19.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:19.477244 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:19.741 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:07:19.742 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:07:19.743 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:19.743 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:19.743 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:19.744 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:19.744 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:19.744 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:19.747 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:07:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:20.481021 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:20.481338 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:21.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 06:07:21 dist-test-slave-pgkr krb5kdc[2506](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786428441, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.111.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:07:21.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:21.086292 2917 init.cc:295] Successfully reacquired a new kerberos TGT
06:07:21.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:21.472695 2902 master_runner.cc:186] Error getting master registration for 127.2.111.62:45719: Network error: Client connection negotiation failed: client connection to 127.2.111.62:45719: connect: Connection refused (error 111),
06:07:21.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:21.473084 2902 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:07:21.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:21.484009 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:21.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:21.484229 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:22.486758 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:22.486989 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:23.489603 2994 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e27f24c0719a4d6a90d89d362dde86f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:07:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 06:07:23.489949 2994 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:07:23.701 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.61:37299(127.2.111.61:37299)] connecting to peer
06:07:23.702 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.111.62:45719(127.2.111.62:45719)] connecting to peer
06:07:23.703 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:23.703 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:23.704 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.61:37299(127.2.111.61:37299)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:23.704 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.111.62:45719(127.2.111.62:45719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:23.704 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.62:45719: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719
06:07:23.704 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.111.61:37299: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
06:07:23.706 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719; will retry
06:07:23.710 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29907), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [191ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [194ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [563ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [574ms] refreshing cache from master, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [583ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [588ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [593ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [594ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [614ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [619ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [621ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [624ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [634ms] refreshing cache from master, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [638ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [639ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [641ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [680ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [681ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [683ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [719ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [720ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [721ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [724ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [780ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [781ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [782ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [920ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [920ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [922ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [925ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1174ms] refreshing cache from master, [1175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [1179ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1187ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1314ms] refreshing cache from master, [1314ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [1315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [1321ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1575ms] refreshing cache from master, [1575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [1577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [1581ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1581ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1583ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [2954ms] refreshing cache from master, [2955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [2955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [2956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [2958ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [2958ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [2960ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [2960ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [4215ms] refreshing cache from master, [4215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [4216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [4217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [4219ms] too many traces: truncated at 100 traces, deferred=Deferred@1102704151(state=PENDING, result=null, callback=(continuation of Deferred@252281418 after retry RPC after error@894743980) -> (continuation of Deferred@817290772 after retry RPC after error@1047615299) -> (continuation of Deferred@2099874069 after retry RPC after error@1832958665) -> (continuation of Deferred@1795198731 after retry RPC after error@1610548839) -> (continuation of Deferred@1848593836 after retry RPC after error@114373714) -> (continuation of Deferred@1774619476 after retry RPC after error@1319986229) -> (continuation of Deferred@1518010977 after retry RPC after error@1426575064) -> (continuation of Deferred@138830599 after retry RPC after error@770646774) -> (continuation of Deferred@797263136 after retry RPC after error@1142898438) -> (continuation of Deferred@2138710745 after retry RPC after error@735467346) -> (continuation of Deferred@780118861 after retry RPC after error@329602699) -> (continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417), errback=(continuation of Deferred@252281418 after retry RPC after error@894743980) -> (continuation of Deferred@817290772 after retry RPC after error@1047615299) -> (continuation of Deferred@2099874069 after retry RPC after error@1832958665) -> (continuation of Deferred@1795198731 after retry RPC after error@1610548839) -> (continuation of Deferred@1848593836 after retry RPC after error@114373714) -> (continuation of Deferred@1774619476 after retry RPC after error@1319986229) -> (continuation of Deferred@1518010977 after retry RPC after error@1426575064) -> (continuation of Deferred@138830599 after retry RPC after error@770646774) -> (continuation of Deferred@797263136 after retry RPC after error@1142898438) -> (continuation of Deferred@2138710745 after retry RPC after error@735467346) -> (continuation of Deferred@780118861 after retry RPC after error@329602699) -> (continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417)))
06:07:23.715 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.111.61:37299(127.2.111.61:37299)] disconnecting while in state TERMINATED
06:07:23.716 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.111.62:45719(127.2.111.62:45719)] disconnecting while in state TERMINATED
06:07:23.716 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.111.60:35377(127.2.111.60:35377)] disconnecting while in state READY
06:07:23.722 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:07:23.724 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:07:23.725 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.2.111.60:35377(127.2.111.60:35377)] handling channelInactive
06:07:23.727 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.111.60:35377(127.2.111.60:35377)] cleaning up while in state READY due to: connection closed
06:07:23.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:23.728979 2492 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu with pid 2902
06:07:24.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T06:07:24Z chronyd exiting
06:07:25.965 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:07:25.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:25.967044 1678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu with pid 2018
06:07:26.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:26.003813 1678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu with pid 2176
06:07:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:26.037559 1678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu with pid 2341
06:07:26.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:26.071943 1678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu with pid 1690
06:07:26.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:26.111438 1678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu with pid 1770
06:07:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 06:07:26.151117 1678 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoDNYHM/build/tsan/bin/kudu with pid 1848
06:07:27.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T06:07:27Z chronyd exiting
06:07:28.324 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29907), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [191ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [194ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [563ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [574ms] refreshing cache from master, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [583ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [588ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [593ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [594ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [614ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [619ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [621ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [624ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [634ms] refreshing cache from master, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [638ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [639ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [641ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [680ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [681ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [683ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [719ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [720ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [721ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [724ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [780ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [781ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [782ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [920ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [920ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [922ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [925ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1174ms] refreshing cache from master, [1175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [1179ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1187ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1314ms] refreshing cache from master, [1314ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [1315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [1321ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1575ms] refreshing cache from master, [1575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [1577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [1581ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1581ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1583ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [2954ms] refreshing cache from master, [2955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [2955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [2956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [2958ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [2958ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [2960ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [2960ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [4215ms] refreshing cache from master, [4215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [4216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [4217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [4219ms] too many traces: truncated at 100 traces, deferred=Deferred@1102704151(state=PENDING, result=null, callback=(continuation of Deferred@252281418 after retry RPC after error@894743980) -> (continuation of Deferred@817290772 after retry RPC after error@1047615299) -> (continuation of Deferred@2099874069 after retry RPC after error@1832958665) -> (continuation of Deferred@1795198731 after retry RPC after error@1610548839) -> (continuation of Deferred@1848593836 after retry RPC after error@114373714) -> (continuation of Deferred@1774619476 after retry RPC after error@1319986229) -> (continuation of Deferred@1518010977 after retry RPC after error@1426575064) -> (continuation of Deferred@138830599 after retry RPC after error@770646774) -> (continuation of Deferred@797263136 after retry RPC after error@1142898438) -> (continuation of Deferred@2138710745 after retry RPC after error@735467346) -> (continuation of Deferred@780118861 after retry RPC after error@329602699) -> (continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417), errback=(continuation of Deferred@252281418 after retry RPC after error@894743980) -> (continuation of Deferred@817290772 after retry RPC after error@1047615299) -> (continuation of Deferred@2099874069 after retry RPC after error@1832958665) -> (continuation of Deferred@1795198731 after retry RPC after error@1610548839) -> (continuation of Deferred@1848593836 after retry RPC after error@114373714) -> (continuation of Deferred@1774619476 after retry RPC after error@1319986229) -> (continuation of Deferred@1518010977 after retry RPC after error@1426575064) -> (continuation of Deferred@138830599 after retry RPC after error@770646774) -> (continuation of Deferred@797263136 after retry RPC after error@1142898438) -> (continuation of Deferred@2138710745 after retry RPC after error@735467346) -> (continuation of Deferred@780118861 after retry RPC after error@329602699) -> (continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.111.61:37299
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29907), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [191ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [194ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [563ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [574ms] refreshing cache from master, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [583ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [588ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [593ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [594ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [614ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [619ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [621ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [624ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [634ms] refreshing cache from master, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [638ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [639ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [641ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [680ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [681ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [683ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [719ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [720ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [721ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [724ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [780ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [781ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [782ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [920ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [920ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [922ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [925ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1174ms] refreshing cache from master, [1175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [1179ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [1185ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1187ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1314ms] refreshing cache from master, [1314ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [1315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [1321ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1575ms] refreshing cache from master, [1575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [1576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [1577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [1581ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [1581ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [1582ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [1583ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [2954ms] refreshing cache from master, [2955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [2955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [2956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [2958ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.62:45719: Network error: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719, [2958ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.61:37299: Network error: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [2960ms] Sub RPC ConnectToMaster: received response from server master-127.2.111.60:35377: OK, [2960ms] delaying RPC due to: Service unavailable: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299, [4215ms] refreshing cache from master, [4215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.61:37299, [4216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.60:35377, [4217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.111.62:45719, [4219ms] too many traces: truncated at 100 traces, deferred=Deferred@1102704151(state=PENDING, result=null, callback=(continuation of Deferred@252281418 after retry RPC after error@894743980) -> (continuation of Deferred@817290772 after retry RPC after error@1047615299) -> (continuation of Deferred@2099874069 after retry RPC after error@1832958665) -> (continuation of Deferred@1795198731 after retry RPC after error@1610548839) -> (continuation of Deferred@1848593836 after retry RPC after error@114373714) -> (continuation of Deferred@1774619476 after retry RPC after error@1319986229) -> (continuation of Deferred@1518010977 after retry RPC after error@1426575064) -> (continuation of Deferred@138830599 after retry RPC after error@770646774) -> (continuation of Deferred@797263136 after retry RPC after error@1142898438) -> (continuation of Deferred@2138710745 after retry RPC after error@735467346) -> (continuation of Deferred@780118861 after retry RPC after error@329602699) -> (continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417), errback=(continuation of Deferred@252281418 after retry RPC after error@894743980) -> (continuation of Deferred@817290772 after retry RPC after error@1047615299) -> (continuation of Deferred@2099874069 after retry RPC after error@1832958665) -> (continuation of Deferred@1795198731 after retry RPC after error@1610548839) -> (continuation of Deferred@1848593836 after retry RPC after error@114373714) -> (continuation of Deferred@1774619476 after retry RPC after error@1319986229) -> (continuation of Deferred@1518010977 after retry RPC after error@1426575064) -> (continuation of Deferred@138830599 after retry RPC after error@770646774) -> (continuation of Deferred@797263136 after retry RPC after error@1142898438) -> (continuation of Deferred@2138710745 after retry RPC after error@735467346) -> (continuation of Deferred@780118861 after retry RPC after error@329602699) -> (continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.111.61:37299,127.2.111.60:35377,127.2.111.62:45719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.62:45719(127.2.111.62:45719): Connection refused: /127.2.111.62:45719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.111.61:37299(127.2.111.61:37299): Connection refused: /127.2.111.61:37299
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.111.61:37299
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)