01:43:53.952 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
01:43:53.965 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoRo5Ty/test-tmp
01:43:53.967 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:43:53.971 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoRo5Ty/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:43:54.240 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
01:43:54.398 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:43:54.399 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:43:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:54.403405 1594 mini_oidc.cc:152] Starting JWKS server
01:43:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:54.407189 1594 webserver.cc:460] Webserver started at http://127.0.0.1:43771/ using document root <none> and password file <none>
01:43:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:54.407753 1594 mini_oidc.cc:189] Starting OIDC Discovery server
01:43:54.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:54.410020 1594 webserver.cc:460] Webserver started at http://127.0.0.1:36033/ using document root <none> and password file <none>
01:43:54.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:43:54Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:43:54.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:43:54Z Disabled control of system clock
01:43:54.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:54.461716 1594 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:43:54.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:43:54.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/wal
01:43:54.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/data
01:43:54.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:43:54.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:43:54.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:43:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:43:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:43:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:43:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/logs
01:43:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/data/info.pb
01:43:54.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:43:54.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:43:54.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:43:54.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:43:54.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:43:54.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:43:54.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:43:54.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:43:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:43:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:43:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:43:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:43:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.190:38679
01:43:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.190
01:43:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:43:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:43:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:43:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:43:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:43:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:43:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:43:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:43:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43771/jwks.json with env {}
01:43:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.766269 1606 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:43:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.766832 1606 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:43:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.767452 1606 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:43:54.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.797820 1606 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:43:54.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.798103 1606 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:43:54.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.798354 1606 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:43:54.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.798549 1606 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:43:54.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.798828 1606 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:43:54.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.799064 1606 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43771/jwks.json
01:43:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:54.833985 1606 master_runner.cc:386] Master server non-default flags:
01:43:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:43:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:43:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:43:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:43:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/data
01:43:54.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/wal
01:43:54.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:43:54.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:43:54.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:43:54.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:43:54.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:43:54.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.190:38679
01:43:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:43:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:43:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:43:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43771/jwks.json
01:43:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:43:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:43:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/data/info.pb
01:43:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.190
01:43:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:43:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/logs
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:43:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:43:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:43:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:43:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:43:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:43:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:54.835197 1606 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:43:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:54.837468 1606 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:43:54.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.861475 1612 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:43:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.861465 1615 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:43:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:54.861544 1613 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:43:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:54.861955 1606 server_base.cc:1034] running on GCE node
01:43:56.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.123637 1606 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:43:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.127368 1606 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:43:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.129237 1606 hybrid_clock.cc:648] HybridClock initialized: now 1781919836129180 us; error 61 us; skew 500 ppm
01:43:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.130367 1606 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:43:56.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.136873 1606 webserver.cc:460] Webserver started at http://127.1.142.190:41565/ using document root <none> and password file <none>
01:43:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.137814 1606 fs_manager.cc:359] Metadata directory not provided
01:43:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.138005 1606 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:43:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.138396 1606 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:43:56.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.143131 1606 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/data/instance:
01:43:56.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57680b2675ec4564a52fd9677b5f0cfd"
01:43:56.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:43:56 on dist-test-slave-6rcr"
01:43:56.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.144299 1606 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/wal/instance:
01:43:56.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57680b2675ec4564a52fd9677b5f0cfd"
01:43:56.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:43:56 on dist-test-slave-6rcr"
01:43:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.151795 1606 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
01:43:56.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.157223 1622 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:43:56.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.158422 1606 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
01:43:56.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.158752 1606 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/wal
01:43:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57680b2675ec4564a52fd9677b5f0cfd"
01:43:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:43:56 on dist-test-slave-6rcr"
01:43:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.159061 1606 fs_report.cc:383] FS layout report
01:43:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:43:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/wal
01:43:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/wal
01:43:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/data/data
01:43:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:43:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:43:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:43:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:43:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:43:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:43:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:43:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:43:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:43:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:43:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:43:56.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.212059 1606 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:43:56.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.213532 1606 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:43:56.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.213935 1606 kserver.cc:163] Server-wide thread pool size limit: 3276
01:43:56.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.289454 1606 rpc_server.cc:307] RPC server started. Bound to: 127.1.142.190:38679
01:43:56.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.289538 1673 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.142.190:38679 every 8 connection(s)
01:43:56.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.294020 1606 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/data/info.pb
01:43:56.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.298484 1594 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 1606
01:43:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.299036 1594 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-0/wal/instance
01:43:56.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.301141 1674 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:43:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.304170 1594 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:43:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:43:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/wal
01:43:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/data
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/logs
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/data/info.pb
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:43:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:43:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:43:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:43:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:43:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:43:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:43:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:43:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:43:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:43:56.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:43:56.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.189:38271
01:43:56.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.189
01:43:56.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:43:56.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:43:56.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:43:56.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:43:56.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:43:56.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:43:56.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:43:56.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:43:56.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43771/jwks.json with env {}
01:43:56.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.316780 1674 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } has no permanent_uuid. Determining permanent_uuid...
01:43:56.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.347831 1674 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } has no permanent_uuid. Determining permanent_uuid...
01:43:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.350059 1624 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.142.189:38271: connect: Connection refused (error 111)
01:43:56.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.354625 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.189:38271: Network error: Client connection negotiation failed: client connection to 127.1.142.189:38271: connect: Connection refused (error 111)
01:43:56.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.404088 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } attempt: 1
01:43:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.408279 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.189:38271: Network error: Client connection negotiation failed: client connection to 127.1.142.189:38271: connect: Connection refused (error 111)
01:43:56.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.476713 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } attempt: 2
01:43:56.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.480665 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.189:38271: Network error: Client connection negotiation failed: client connection to 127.1.142.189:38271: connect: Connection refused (error 111)
01:43:56.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.572101 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } attempt: 3
01:43:56.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.576195 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.189:38271: Network error: Client connection negotiation failed: client connection to 127.1.142.189:38271: connect: Connection refused (error 111)
01:43:56.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.622243 1676 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:43:56.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.622789 1676 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:43:56.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.623309 1676 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:43:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.653924 1676 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:43:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.654198 1676 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:43:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.654448 1676 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:43:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.654682 1676 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:43:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.654865 1676 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:43:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.654994 1676 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43771/jwks.json
01:43:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.690860 1676 master_runner.cc:386] Master server non-default flags:
01:43:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:43:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:43:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:43:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:43:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/data
01:43:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/wal
01:43:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:43:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:43:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:43:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:43:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:43:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.189:38271
01:43:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:43:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:43:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:43:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43771/jwks.json
01:43:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:43:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:43:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/data/info.pb
01:43:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.189
01:43:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:43:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:43:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:43:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:43:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:43:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:43:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/logs
01:43:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:43:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:43:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:43:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:43:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:43:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:43:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:43:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:43:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:43:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:43:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.692087 1676 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:43:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.693845 1676 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:43:56.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.714331 1685 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:43:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.719583 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } attempt: 4
01:43:56.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.724098 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.189:38271: Network error: Client connection negotiation failed: client connection to 127.1.142.189:38271: connect: Connection refused (error 111)
01:43:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.023571 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } attempt: 5
01:43:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:57.027486 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.189:38271: Network error: Client connection negotiation failed: client connection to 127.1.142.189:38271: connect: Connection refused (error 111)
01:43:57.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.575160 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } attempt: 6
01:43:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:57.582157 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.189:38271: Network error: Client connection negotiation failed: client connection to 127.1.142.189:38271: connect: Connection refused (error 111)
01:43:57.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.714574 1688 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:43:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:56.716147 1676 server_base.cc:1034] running on GCE node
01:43:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:56.714502 1686 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:43:57.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.952256 1676 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:43:57.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.956233 1676 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:43:57.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.957717 1676 hybrid_clock.cc:648] HybridClock initialized: now 1781919837957667 us; error 69 us; skew 500 ppm
01:43:57.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.958767 1676 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:43:57.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.966224 1676 webserver.cc:460] Webserver started at http://127.1.142.189:40631/ using document root <none> and password file <none>
01:43:57.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.967203 1676 fs_manager.cc:359] Metadata directory not provided
01:43:57.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.967383 1676 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:43:57.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.967778 1676 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:43:57.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.972648 1676 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/data/instance:
01:43:57.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5613ef268884b418d4858fe9b024ac4"
01:43:57.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:43:57 on dist-test-slave-6rcr"
01:43:57.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.973851 1676 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/wal/instance:
01:43:57.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5613ef268884b418d4858fe9b024ac4"
01:43:57.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:43:57 on dist-test-slave-6rcr"
01:43:57.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.980875 1676 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
01:43:57.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.986634 1697 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:43:57.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.987753 1676 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
01:43:57.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.988140 1676 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/wal
01:43:57.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d5613ef268884b418d4858fe9b024ac4"
01:43:57.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:43:57 on dist-test-slave-6rcr"
01:43:57.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:57.988525 1676 fs_report.cc:383] FS layout report
01:43:57.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:43:57.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/wal
01:43:57.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/wal
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/data/data
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:43:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:43:58.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.026156 1676 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:43:58.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.028120 1676 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:43:58.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.028633 1676 kserver.cc:163] Server-wide thread pool size limit: 3276
01:43:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.104378 1676 rpc_server.cc:307] RPC server started. Bound to: 127.1.142.189:38271
01:43:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.104494 1748 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.142.189:38271 every 8 connection(s)
01:43:58.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.107283 1676 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/data/info.pb
01:43:58.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.112861 1594 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 1676
01:43:58.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.113344 1594 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-1/wal/instance
01:43:58.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.114473 1749 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:43:58.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.116366 1594 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:43:58.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:43:58.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/wal
01:43:58.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/data
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/logs
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/data/info.pb
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.188:42183
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.188
01:43:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:43:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:43:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:43:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:43:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:43:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:43:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:43:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:43:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43771/jwks.json with env {}
01:43:58.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.132735 1749 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } has no permanent_uuid. Determining permanent_uuid...
01:43:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.156847 1749 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } has no permanent_uuid. Determining permanent_uuid...
01:43:58.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.173372 1749 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } has no permanent_uuid. Determining permanent_uuid...
01:43:58.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.175802 1698 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:58.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.179674 1749 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:58.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.229220 1749 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 1
01:43:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.234545 1749 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.302973 1749 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 2
01:43:58.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.306748 1749 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.398169 1749 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 3
01:43:58.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.401979 1749 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:58.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.446272 1751 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:43:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.446825 1751 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:43:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.447603 1751 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:43:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.478803 1751 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:43:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.479089 1751 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:43:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.479281 1751 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:43:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.479457 1751 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:43:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.479633 1751 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:43:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.479774 1751 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43771/jwks.json
01:43:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.514956 1751 master_runner.cc:386] Master server non-default flags:
01:43:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:43:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:43:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:43:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:43:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/data
01:43:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/wal
01:43:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.188:42183
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43771/jwks.json
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/data/info.pb
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.188
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:43:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/logs
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:43:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:43:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.516265 1751 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:43:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.518189 1751 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:43:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.538472 1764 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:43:58.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.545404 1749 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 4
01:43:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.549265 1749 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.642758 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } attempt: 7
01:43:58.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.661962 1674 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } has no permanent_uuid. Determining permanent_uuid...
01:43:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.669236 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:58.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.727720 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 1
01:43:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.733121 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:58.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.814630 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 2
01:43:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.819440 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.848724 1749 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 5
01:43:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.853783 1749 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.904847 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 3
01:43:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.909888 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:59.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.050290 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 4
01:43:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:59.054941 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.338361 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 5
01:43:59.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:59.343210 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.401201 1749 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 6
01:43:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:59.408565 1749 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:59.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.538496 1762 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:43:59.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:58.538702 1761 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:43:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:58.541878 1751 server_base.cc:1034] running on GCE node
01:43:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.759598 1751 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:43:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.762840 1751 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:43:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.764269 1751 hybrid_clock.cc:648] HybridClock initialized: now 1781919839764228 us; error 62 us; skew 500 ppm
01:43:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.765045 1751 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:43:59.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.772183 1751 webserver.cc:460] Webserver started at http://127.1.142.188:34867/ using document root <none> and password file <none>
01:43:59.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.773072 1751 fs_manager.cc:359] Metadata directory not provided
01:43:59.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.773249 1751 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:43:59.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.773715 1751 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:43:59.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.780674 1751 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/data/instance:
01:43:59.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1591f7ccb7746348ef23d0ddd162e56"
01:43:59.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:43:59 on dist-test-slave-6rcr"
01:43:59.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.782303 1751 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/wal/instance:
01:43:59.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1591f7ccb7746348ef23d0ddd162e56"
01:43:59.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:43:59 on dist-test-slave-6rcr"
01:43:59.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.792279 1751 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
01:43:59.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.799983 1775 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:43:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.801324 1751 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.801666 1751 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/wal
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1591f7ccb7746348ef23d0ddd162e56"
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:43:59 on dist-test-slave-6rcr"
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.801985 1751 fs_report.cc:383] FS layout report
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/wal
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/wal
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/data/data
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:43:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:43:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.840639 1751 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:43:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.842521 1751 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:43:59.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.842934 1751 kserver.cc:163] Server-wide thread pool size limit: 3276
01:43:59.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.895629 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 6
01:43:59.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:43:59.902916 1674 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.142.188:42183: Network error: Client connection negotiation failed: client connection to 127.1.142.188:42183: connect: Connection refused (error 111)
01:43:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.921067 1751 rpc_server.cc:307] RPC server started. Bound to: 127.1.142.188:42183
01:43:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.921104 1828 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.142.188:42183 every 8 connection(s)
01:43:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.924052 1751 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/data/info.pb
01:43:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.930776 1829 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:43:59.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.934301 1594 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 1751
01:43:59.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.934711 1594 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/master-2/wal/instance
01:43:59.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.944448 1829 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } has no permanent_uuid. Determining permanent_uuid...
01:43:59.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.972883 1829 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } has no permanent_uuid. Determining permanent_uuid...
01:43:59.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:43:59.987489 1829 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } has no permanent_uuid. Determining permanent_uuid...
01:44:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.007596 1829 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56: Bootstrap starting.
01:44:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.015226 1829 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56: Neither blocks nor log segments found. Creating new log.
01:44:00.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.017506 1829 log.cc:824] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56: Log is configured to *not* fsync() on all Append() calls
01:44:00.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.024252 1829 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56: No bootstrap required, opened a new log
01:44:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.049827 1829 raft_consensus.cc:348] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } }
01:44:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.050447 1829 raft_consensus.cc:374] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:44:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.050673 1829 raft_consensus.cc:729] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e1591f7ccb7746348ef23d0ddd162e56, State: Initialized, Role: FOLLOWER
01:44:00.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.051318 1829 consensus_queue.cc:260] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } }
01:44:00.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.054061 1838 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } } }
01:44:00.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.055186 1838 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [sys.catalog]: This master's current role is: FOLLOWER
01:44:00.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.056010 1829 sys_catalog.cc:564] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [sys.catalog]: configured and running, proceeding with master startup.
01:44:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:00.082744 1849 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:00.083105 1849 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:00.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:00.098368 1781 tablet.cc:2369] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:44:00.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.079223 1751 master_runner.cc:186] Error getting master registration for 127.1.142.190:38679: OK, instance_id { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" instance_seqno: 1781919836269497 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:44:00.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.115430 1751 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:44:00.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.314054 1838 raft_consensus.cc:482] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:44:00.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.314625 1838 raft_consensus.cc:504] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } }
01:44:00.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.317060 1838 leader_election.cc:288] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 57680b2675ec4564a52fd9677b5f0cfd (127.1.142.190:38679), d5613ef268884b418d4858fe9b024ac4 (127.1.142.189:38271)
01:44:00.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.468984 1749 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 7
01:44:00.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.494385 1749 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4: Bootstrap starting.
01:44:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.502151 1749 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4: Neither blocks nor log segments found. Creating new log.
01:44:00.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.504467 1749 log.cc:824] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4: Log is configured to *not* fsync() on all Append() calls
01:44:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.510550 1749 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4: No bootstrap required, opened a new log
01:44:00.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.542841 1749 raft_consensus.cc:348] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } }
01:44:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.543723 1749 raft_consensus.cc:374] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:44:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.543995 1749 raft_consensus.cc:729] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d5613ef268884b418d4858fe9b024ac4, State: Initialized, Role: FOLLOWER
01:44:00.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.544988 1749 consensus_queue.cc:260] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } }
01:44:00.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.548312 1853 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } } }
01:44:00.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.550242 1853 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [sys.catalog]: This master's current role is: FOLLOWER
01:44:00.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.550593 1749 sys_catalog.cc:564] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [sys.catalog]: configured and running, proceeding with master startup.
01:44:00.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.550820 1724 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e1591f7ccb7746348ef23d0ddd162e56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5613ef268884b418d4858fe9b024ac4" is_pre_election: true
01:44:00.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.551644 1724 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e1591f7ccb7746348ef23d0ddd162e56 in term 0.
01:44:00.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.552960 1777 leader_election.cc:302] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d5613ef268884b418d4858fe9b024ac4, e1591f7ccb7746348ef23d0ddd162e56; no voters:
01:44:00.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.554265 1838 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 0 FOLLOWER]: Leader pre-election won for term 1
01:44:00.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.554603 1838 raft_consensus.cc:482] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:44:00.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.554934 1838 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 0 FOLLOWER]: Advancing to term 1
01:44:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.563047 1838 raft_consensus.cc:504] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } }
01:44:00.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.565779 1838 leader_election.cc:288] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [CANDIDATE]: Term 1 election: Requested vote from peers 57680b2675ec4564a52fd9677b5f0cfd (127.1.142.190:38679), d5613ef268884b418d4858fe9b024ac4 (127.1.142.189:38271)
01:44:00.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.566694 1724 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e1591f7ccb7746348ef23d0ddd162e56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d5613ef268884b418d4858fe9b024ac4"
01:44:00.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.567250 1724 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [term 0 FOLLOWER]: Advancing to term 1
01:44:00.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.574390 1724 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e1591f7ccb7746348ef23d0ddd162e56 in term 1.
01:44:00.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.575975 1777 leader_election.cc:302] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d5613ef268884b418d4858fe9b024ac4, e1591f7ccb7746348ef23d0ddd162e56; no voters:
01:44:00.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.576746 1838 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 1 FOLLOWER]: Leader election won for term 1
01:44:00.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.578724 1838 raft_consensus.cc:686] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [term 1 LEADER]: Becoming Leader. State: Replica: e1591f7ccb7746348ef23d0ddd162e56, State: Running, Role: LEADER
01:44:00.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.579556 1838 consensus_queue.cc:237] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } }
01:44:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.618322 1861 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e1591f7ccb7746348ef23d0ddd162e56. Latest consensus state: current_term: 1 leader_uuid: "e1591f7ccb7746348ef23d0ddd162e56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } } }
01:44:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.621054 1861 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [sys.catalog]: This master's current role is: LEADER
01:44:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:00.627173 1866 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:00.627843 1866 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.640086 1868 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:44:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.607416 1676 master_runner.cc:186] Error getting master registration for 127.1.142.190:38679: OK, instance_id { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" instance_seqno: 1781919836269497 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:44:00.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.640810 1676 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:44:00.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.653806 1868 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:44:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.678340 1724 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [term 1 FOLLOWER]: Refusing update from remote peer e1591f7ccb7746348ef23d0ddd162e56: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:44:00.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.679965 1861 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:44:00.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.716351 1853 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e1591f7ccb7746348ef23d0ddd162e56. Latest consensus state: current_term: 1 leader_uuid: "e1591f7ccb7746348ef23d0ddd162e56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } } }
01:44:00.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.717119 1853 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [sys.catalog]: This master's current role is: FOLLOWER
01:44:00.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.727614 1861 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e1591f7ccb7746348ef23d0ddd162e56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } } }
01:44:00.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.728410 1861 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [sys.catalog]: This master's current role is: LEADER
01:44:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.735711 1853 sys_catalog.cc:455] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e1591f7ccb7746348ef23d0ddd162e56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } } }
01:44:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.736433 1853 sys_catalog.cc:458] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4 [sys.catalog]: This master's current role is: FOLLOWER
01:44:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.739985 1868 catalog_manager.cc:1347] Generated new cluster ID: 74f290bdca6842488ceaf37c90cc6dcb
01:44:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.740291 1868 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:44:00.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.768401 1868 catalog_manager.cc:1370] Generated new certificate authority record
01:44:00.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.772480 1868 catalog_manager.cc:1504] Loading token signing keys...
01:44:00.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.797755 1868 catalog_manager.cc:5868] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56: Generated new TSK 0
01:44:00.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.799356 1868 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:44:00.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.821238 1594 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:00.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/wal
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/data
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/logs
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/data/info.pb
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:00.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.129:0
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.142.129
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.129
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:44:00.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:00.936281 1674 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } attempt: 7
01:44:01.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.029075 1674 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd: Bootstrap starting.
01:44:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.037315 1674 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd: Neither blocks nor log segments found. Creating new log.
01:44:01.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.040674 1674 log.cc:824] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd: Log is configured to *not* fsync() on all Append() calls
01:44:01.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.064731 1674 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd: No bootstrap required, opened a new log
01:44:01.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.187899 1674 raft_consensus.cc:348] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } }
01:44:01.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.188845 1674 raft_consensus.cc:374] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:44:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.189100 1674 raft_consensus.cc:729] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 57680b2675ec4564a52fd9677b5f0cfd, State: Initialized, Role: FOLLOWER
01:44:01.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.207983 1674 consensus_queue.cc:260] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } }
01:44:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.215975 1877 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } } }
01:44:01.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.216806 1877 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [sys.catalog]: This master's current role is: FOLLOWER
01:44:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.219602 1674 sys_catalog.cc:564] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [sys.catalog]: configured and running, proceeding with master startup.
01:44:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.219748 1649 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e1591f7ccb7746348ef23d0ddd162e56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57680b2675ec4564a52fd9677b5f0cfd" is_pre_election: true
01:44:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.220649 1649 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e1591f7ccb7746348ef23d0ddd162e56 in term 0.
01:44:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.221691 1648 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e1591f7ccb7746348ef23d0ddd162e56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "57680b2675ec4564a52fd9677b5f0cfd"
01:44:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.222713 1648 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [term 0 FOLLOWER]: Advancing to term 1
01:44:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:01.226307 1628 tablet.cc:2369] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:44:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.232692 1648 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e1591f7ccb7746348ef23d0ddd162e56 in term 1.
01:44:01.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.234730 1647 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [term 1 FOLLOWER]: Refusing update from remote peer e1591f7ccb7746348ef23d0ddd162e56: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:44:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.242506 1838 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [LEADER]: Connected to new peer: Peer: permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:44:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.328756 1877 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [sys.catalog]: SysCatalogTable state changed. Reason: New leader e1591f7ccb7746348ef23d0ddd162e56. Latest consensus state: current_term: 1 leader_uuid: "e1591f7ccb7746348ef23d0ddd162e56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } } }
01:44:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.329734 1877 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [sys.catalog]: This master's current role is: FOLLOWER
01:44:01.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.331218 1883 sys_catalog.cc:455] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e1591f7ccb7746348ef23d0ddd162e56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } } }
01:44:01.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.331960 1883 sys_catalog.cc:458] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd [sys.catalog]: This master's current role is: FOLLOWER
01:44:01.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.368901 1893 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:44:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:01.501634 1874 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:44:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:01.502192 1874 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:44:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:01.502875 1874 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:44:01.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:01.546195 1874 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:44:01.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:01.547583 1874 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.142.129
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.609618 1874 tablet_server_runner.cc:78] Tablet server non-default flags:
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/data
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/wal
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.129:0
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:44:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/data/info.pb
01:44:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.129
01:44:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:44:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:44:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:44:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:44:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.142.129
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/logs
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:44:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.611202 1874 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:44:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.613385 1874 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:44:01.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.634945 1866 catalog_manager.cc:1259] Loaded cluster ID: 74f290bdca6842488ceaf37c90cc6dcb
01:44:01.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.635643 1866 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4: loading cluster ID for follower catalog manager: success
01:44:01.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:01.641669 1899 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:01.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.656625 1866 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4: acquiring CA information for follower catalog manager: success
01:44:01.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.664279 1866 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d5613ef268884b418d4858fe9b024ac4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:44:01.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.685441 1838 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e1591f7ccb7746348ef23d0ddd162e56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "57680b2675ec4564a52fd9677b5f0cfd" member_type: VOTER last_known_addr { host: "127.1.142.190" port: 38679 } } peers { permanent_uuid: "d5613ef268884b418d4858fe9b024ac4" member_type: VOTER last_known_addr { host: "127.1.142.189" port: 38271 } } peers { permanent_uuid: "e1591f7ccb7746348ef23d0ddd162e56" member_type: VOTER last_known_addr { host: "127.1.142.188" port: 42183 } } }
01:44:01.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:01.686579 1838 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1591f7ccb7746348ef23d0ddd162e56 [sys.catalog]: This master's current role is: LEADER
01:44:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:02.384768 1893 catalog_manager.cc:1259] Loaded cluster ID: 74f290bdca6842488ceaf37c90cc6dcb
01:44:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:02.385133 1893 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd: loading cluster ID for follower catalog manager: success
01:44:02.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:02.426990 1893 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd: acquiring CA information for follower catalog manager: success
01:44:02.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:02.452876 1893 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 57680b2675ec4564a52fd9677b5f0cfd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:44:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:01.642792 1900 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:03.033845 1898 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 1874
01:44:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:03.762195 1898 kernel_stack_watchdog.cc:198] Thread 1874 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 392ms:
01:44:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
01:44:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
01:44:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
01:44:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
01:44:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:03.763072 1874 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 2.124s user 0.773s sys 1.202s
01:44:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:03.763409 1874 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 2.125s user 0.773s sys 1.202s
01:44:03.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:03.778999 1910 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:03.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.779637 1874 server_base.cc:1034] running on GCE node
01:44:03.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.781263 1874 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:44:03.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.784091 1874 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:44:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.785512 1874 hybrid_clock.cc:648] HybridClock initialized: now 1781919843785457 us; error 46 us; skew 500 ppm
01:44:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.786613 1874 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:44:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.796653 1874 webserver.cc:460] Webserver started at http://127.1.142.129:38285/ using document root <none> and password file <none>
01:44:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.797964 1874 fs_manager.cc:359] Metadata directory not provided
01:44:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.798261 1874 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:44:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.798843 1874 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:44:03.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.806247 1874 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/data/instance:
01:44:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfb61ddbc1484b06b095b128bdfe6191"
01:44:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:03 on dist-test-slave-6rcr"
01:44:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.807952 1874 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/wal/instance:
01:44:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfb61ddbc1484b06b095b128bdfe6191"
01:44:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:03 on dist-test-slave-6rcr"
01:44:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.818518 1874 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.006s sys 0.006s
01:44:03.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.827157 1916 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:44:03.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.830386 1874 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.001s sys 0.004s
01:44:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.830837 1874 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/wal
01:44:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfb61ddbc1484b06b095b128bdfe6191"
01:44:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:03 on dist-test-slave-6rcr"
01:44:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.831312 1874 fs_report.cc:383] FS layout report
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/wal
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/wal
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/data/data
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:44:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:44:03.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.888047 1874 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:44:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.889986 1874 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:44:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.890578 1874 kserver.cc:163] Server-wide thread pool size limit: 3276
01:44:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.894335 1874 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:44:03.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.906669 1874 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:44:03.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.906991 1874 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:44:03.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.907284 1874 ts_tablet_manager.cc:610] Registered 0 tablets
01:44:03.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:03.907618 1874 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:44:04.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.210542 1874 rpc_server.cc:307] RPC server started. Bound to: 127.1.142.129:38821
01:44:04.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.210669 2030 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.142.129:38821 every 8 connection(s)
01:44:04.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.214222 1874 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/data/info.pb
01:44:04.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.223248 1594 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 1874
01:44:04.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.223838 1594 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-0/wal/instance
01:44:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.239535 1594 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/wal
01:44:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/data
01:44:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:44:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:44:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:44:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/logs
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/data/info.pb
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.130:0
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.142.130
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.130
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:44:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.329422 2031 heartbeater.cc:347] Connected to a master server at 127.1.142.188:42183
01:44:04.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.329980 2031 heartbeater.cc:464] Registering TS with master...
01:44:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.332050 2031 heartbeater.cc:511] Master 127.1.142.188:42183 requested a full tablet report, sending...
01:44:04.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.335335 1792 ts_manager.cc:194] Registered new tserver with Master: bfb61ddbc1484b06b095b128bdfe6191 (127.1.142.129:38821)
01:44:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.336009 2036 heartbeater.cc:347] Connected to a master server at 127.1.142.190:38679
01:44:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.336246 2032 heartbeater.cc:347] Connected to a master server at 127.1.142.189:38271
01:44:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.336370 2036 heartbeater.cc:464] Registering TS with master...
01:44:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.336617 2032 heartbeater.cc:464] Registering TS with master...
01:44:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.337250 2036 heartbeater.cc:511] Master 127.1.142.190:38679 requested a full tablet report, sending...
01:44:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.337494 2032 heartbeater.cc:511] Master 127.1.142.189:38271 requested a full tablet report, sending...
01:44:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.338410 1792 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.142.129:36451
01:44:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.340687 1714 ts_manager.cc:194] Registered new tserver with Master: bfb61ddbc1484b06b095b128bdfe6191 (127.1.142.129:38821)
01:44:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.340749 1639 ts_manager.cc:194] Registered new tserver with Master: bfb61ddbc1484b06b095b128bdfe6191 (127.1.142.129:38821)
01:44:04.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:04.681826 2035 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:44:04.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:04.682406 2035 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:44:04.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:04.683022 2035 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:44:04.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:04.715164 2035 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:44:04.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:04.716137 2035 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.142.130
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.753999 2035 tablet_server_runner.cc:78] Tablet server non-default flags:
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/data
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/wal
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.130:0
01:44:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:44:04.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:04.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:04.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/data/info.pb
01:44:04.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.130
01:44:04.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:04.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:44:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:44:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:44:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:44:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.142.130
01:44:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/logs
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.755494 2035 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:44:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.757766 2035 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:44:04.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:04.783279 2050 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:05.347061 2031 heartbeater.cc:503] Master 127.1.142.188:42183 was elected leader, sending a full tablet report...
01:44:06.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:04.784505 2049 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:04.785427 2035 server_base.cc:1034] running on GCE node
01:44:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:04.784801 2052 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.089273 2035 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:44:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.092589 2035 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:44:06.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.094107 2035 hybrid_clock.cc:648] HybridClock initialized: now 1781919846094051 us; error 65 us; skew 500 ppm
01:44:06.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.094930 2035 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:44:06.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.102345 2035 webserver.cc:460] Webserver started at http://127.1.142.130:39265/ using document root <none> and password file <none>
01:44:06.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.103650 2035 fs_manager.cc:359] Metadata directory not provided
01:44:06.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.104287 2035 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:44:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.104851 2035 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:44:06.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.112663 2035 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/data/instance:
01:44:06.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7a70d272a084d129b9fcb97ccd95547"
01:44:06.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:06 on dist-test-slave-6rcr"
01:44:06.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.114267 2035 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/wal/instance:
01:44:06.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7a70d272a084d129b9fcb97ccd95547"
01:44:06.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:06 on dist-test-slave-6rcr"
01:44:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.124472 2035 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
01:44:06.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.132593 2059 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:44:06.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.133920 2035 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
01:44:06.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.134305 2035 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/wal
01:44:06.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7a70d272a084d129b9fcb97ccd95547"
01:44:06.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:06 on dist-test-slave-6rcr"
01:44:06.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.134685 2035 fs_report.cc:383] FS layout report
01:44:06.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:44:06.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/wal
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/wal
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/data/data
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:44:06.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:44:06.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.173933 2035 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:44:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.175546 2035 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:44:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.175962 2035 kserver.cc:163] Server-wide thread pool size limit: 3276
01:44:06.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.179268 2035 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:44:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.183490 2035 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:44:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.183677 2035 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:44:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.183869 2035 ts_tablet_manager.cc:610] Registered 0 tablets
01:44:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.183993 2035 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:44:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.360630 2035 rpc_server.cc:307] RPC server started. Bound to: 127.1.142.130:40693
01:44:06.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.360742 2171 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.142.130:40693 every 8 connection(s)
01:44:06.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.363346 2035 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/data/info.pb
01:44:06.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.365954 1594 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 2035
01:44:06.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.367082 1594 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-1/wal/instance
01:44:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.404855 1594 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/wal
01:44:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/data
01:44:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:44:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/logs
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/data/info.pb
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.131:0
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.142.131
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.131
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:06.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:44:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:44:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:44:06.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.491770 2173 heartbeater.cc:347] Connected to a master server at 127.1.142.189:38271
01:44:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.492273 2173 heartbeater.cc:464] Registering TS with master...
01:44:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.492581 2172 heartbeater.cc:347] Connected to a master server at 127.1.142.188:42183
01:44:06.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.492882 2172 heartbeater.cc:464] Registering TS with master...
01:44:06.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.493505 2173 heartbeater.cc:511] Master 127.1.142.189:38271 requested a full tablet report, sending...
01:44:06.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.493790 2172 heartbeater.cc:511] Master 127.1.142.188:42183 requested a full tablet report, sending...
01:44:06.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.496553 1792 ts_manager.cc:194] Registered new tserver with Master: b7a70d272a084d129b9fcb97ccd95547 (127.1.142.130:40693)
01:44:06.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.496723 1714 ts_manager.cc:194] Registered new tserver with Master: b7a70d272a084d129b9fcb97ccd95547 (127.1.142.130:40693)
01:44:06.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.498765 1792 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.142.130:55447
01:44:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.504065 2174 heartbeater.cc:347] Connected to a master server at 127.1.142.190:38679
01:44:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.504356 2174 heartbeater.cc:464] Registering TS with master...
01:44:06.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.504881 2174 heartbeater.cc:511] Master 127.1.142.190:38679 requested a full tablet report, sending...
01:44:06.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.508004 1639 ts_manager.cc:194] Registered new tserver with Master: b7a70d272a084d129b9fcb97ccd95547 (127.1.142.130:40693)
01:44:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:06.793622 2179 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:44:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:06.794085 2179 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:44:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:06.794538 2179 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:44:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:06.827481 2179 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:44:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:06.828291 2179 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.142.131
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.865808 2179 tablet_server_runner.cc:78] Tablet server non-default flags:
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.142.148:36671
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/data
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/wal
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.142.131:0
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/data/info.pb
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.142.131
01:44:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.142.190:38679,127.1.142.189:38271,127.1.142.188:42183
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.142.131
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/logs
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:44:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.867394 2179 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:44:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.869621 2179 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:44:06.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:06.889719 2190 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:07.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:07.502965 2172 heartbeater.cc:503] Master 127.1.142.188:42183 was elected leader, sending a full tablet report...
01:44:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:06.889786 2189 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:06.891434 2179 server_base.cc:1034] running on GCE node
01:44:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:06.889778 2192 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.192412 2179 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:44:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.195135 2179 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:44:08.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.196580 2179 hybrid_clock.cc:648] HybridClock initialized: now 1781919848196548 us; error 41 us; skew 500 ppm
01:44:08.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.197546 2179 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:44:08.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.206987 2179 webserver.cc:460] Webserver started at http://127.1.142.131:41941/ using document root <none> and password file <none>
01:44:08.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.208148 2179 fs_manager.cc:359] Metadata directory not provided
01:44:08.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.208407 2179 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:44:08.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.209000 2179 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:44:08.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.214238 2179 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/data/instance:
01:44:08.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8600601ffbab48ef870f3038158e2ab7"
01:44:08.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:08 on dist-test-slave-6rcr"
01:44:08.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.215590 2179 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/wal/instance:
01:44:08.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8600601ffbab48ef870f3038158e2ab7"
01:44:08.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:08 on dist-test-slave-6rcr"
01:44:08.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.224476 2179 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.002s sys 0.008s
01:44:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.231518 2199 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:44:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.232772 2179 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
01:44:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.233189 2179 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/wal
01:44:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8600601ffbab48ef870f3038158e2ab7"
01:44:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:08 on dist-test-slave-6rcr"
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.233652 2179 fs_report.cc:383] FS layout report
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/wal
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/wal
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/data/data
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:44:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:44:08.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.287266 2179 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:44:08.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.289173 2179 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:44:08.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.289745 2179 kserver.cc:163] Server-wide thread pool size limit: 3276
01:44:08.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.293696 2179 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:44:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.305850 2179 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:44:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.306140 2179 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:44:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.306452 2179 ts_tablet_manager.cc:610] Registered 0 tablets
01:44:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.306634 2179 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:44:08.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.485395 2179 rpc_server.cc:307] RPC server started. Bound to: 127.1.142.131:33663
01:44:08.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.485500 2312 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.142.131:33663 every 8 connection(s)
01:44:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.488224 2179 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/data/info.pb
01:44:08.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.491437 1594 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 2179
01:44:08.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.491899 1594 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster5085284825066466509/ts-2/wal/instance
01:44:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.546943 2313 heartbeater.cc:347] Connected to a master server at 127.1.142.188:42183
01:44:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.547375 2313 heartbeater.cc:464] Registering TS with master...
01:44:08.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.548823 2313 heartbeater.cc:511] Master 127.1.142.188:42183 requested a full tablet report, sending...
01:44:08.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.551973 1792 ts_manager.cc:194] Registered new tserver with Master: 8600601ffbab48ef870f3038158e2ab7 (127.1.142.131:33663)
01:44:08.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.553534 2314 heartbeater.cc:347] Connected to a master server at 127.1.142.189:38271
01:44:08.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.553879 2314 heartbeater.cc:464] Registering TS with master...
01:44:08.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.554349 1792 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.142.131:51149
01:44:08.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.555038 2315 heartbeater.cc:347] Connected to a master server at 127.1.142.190:38679
01:44:08.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.555302 2315 heartbeater.cc:464] Registering TS with master...
01:44:08.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.556074 2315 heartbeater.cc:511] Master 127.1.142.190:38679 requested a full tablet report, sending...
01:44:08.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.557374 2314 heartbeater.cc:511] Master 127.1.142.189:38271 requested a full tablet report, sending...
01:44:08.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.560063 1714 ts_manager.cc:194] Registered new tserver with Master: 8600601ffbab48ef870f3038158e2ab7 (127.1.142.131:33663)
01:44:08.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.560956 1639 ts_manager.cc:194] Registered new tserver with Master: 8600601ffbab48ef870f3038158e2ab7 (127.1.142.131:33663)
01:44:08.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:08.576864 1594 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
01:44:08.588 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:44:08.589 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
01:44:08.601 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.142.190"
port: 38679
}
bound_http_address {
host: "127.1.142.190"
port: 41565
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.142.189"
port: 38271
}
bound_http_address {
host: "127.1.142.189"
port: 40631
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.142.188"
port: 42183
}
bound_http_address {
host: "127.1.142.188"
port: 34867
}
}
}
01:44:08.608 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:44:08.611 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.142.129"
port: 38821
}
bound_http_address {
host: "127.1.142.129"
port: 38285
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.142.130"
port: 40693
}
bound_http_address {
host: "127.1.142.130"
port: 39265
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.142.131"
port: 33663
}
bound_http_address {
host: "127.1.142.131"
port: 41941
}
}
}
01:44:08.612 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
01:44:08.801 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:44:08.912 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoRo5Ty/test-tmp
01:44:08.913 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:44:08.913 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoRo5Ty/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:44:08.916 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
01:44:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:09.295588 2027 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba6c0 after lost signal to thread 1894
01:44:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:09.296446 2027 debug-util.cc:398] Leaking SignalData structure 0x7b08000af8c0 after lost signal to thread 2030
01:44:09.312 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:09.312 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:44:09.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:09.317076 2324 mini_oidc.cc:152] Starting JWKS server
01:44:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:09.322937 2324 webserver.cc:460] Webserver started at http://127.0.0.1:35263/ using document root <none> and password file <none>
01:44:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:09.323515 2324 mini_oidc.cc:189] Starting OIDC Discovery server
01:44:09.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:09.332588 2324 webserver.cc:460] Webserver started at http://127.0.0.1:34465/ using document root <none> and password file <none>
01:44:09.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:09.562268 2313 heartbeater.cc:503] Master 127.1.142.188:42183 was elected leader, sending a full tablet report...
01:44:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
01:44:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/principal' for realm 'KRBTEST.COM',
01:44:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
01:44:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:09 dist-test-slave-6rcr krb5kdc[2337](info): setting up network...
01:44:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
01:44:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:09 dist-test-slave-6rcr krb5kdc[2337](info): set up 2 sockets
01:44:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:09 dist-test-slave-6rcr krb5kdc[2337](info): commencing operation
01:44:09.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
01:44:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:10.873037 2324 mini_kdc.cc:121] Time spent starting KDC: real 1.511s user 0.003s sys 0.022s
01:44:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
01:44:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:44:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
01:44:10.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
01:44:10.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:44:10.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
01:44:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
01:44:10.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:44:10.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
01:44:10.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:44:10.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/test-user.keytab.
01:44:10.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/test-user.keytab.
01:44:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:11 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919851, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:44:11.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:44:11Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:44:11.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:44:11Z Disabled control of system clock
01:44:11.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.69.62@KRBTEST.COM; defaulting to no policy
01:44:11.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:44:11.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.69.62@KRBTEST.COM" created.
01:44:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:44:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.69.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.62.keytab.
01:44:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.69.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.62.keytab.
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:11.094683 2324 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/logs
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data/info.pb
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:44:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.62:46643
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.62
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.62.keytab
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.62
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:44:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.424976 2350 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:44:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.425627 2350 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:44:11.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.426213 2350 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:44:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.457922 2350 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:44:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.458164 2350 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:44:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.458472 2350 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:44:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.458774 2350 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:44:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.459101 2350 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:44:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.459434 2350 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:44:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.459699 2350 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35263/jwks.json
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:11.511772 2350 master_runner.cc:386] Master server non-default flags:
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.62.keytab
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.62
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.62:46643
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data/info.pb
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.62
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:44:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/logs
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:44:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:11.512907 2350 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:44:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:11.513942 2350 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:44:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.534475 2357 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:11.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.708736 2168 debug-util.cc:398] Leaking SignalData structure 0x7b08000af940 after lost signal to thread 2043
01:44:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.709776 2168 debug-util.cc:398] Leaking SignalData structure 0x7b08000af980 after lost signal to thread 2171
01:44:12.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.535045 2360 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:12.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:11.535777 2358 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:11.536861 2350 server_base.cc:1034] running on GCE node
01:44:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.836695 2350 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:44:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.840083 2350 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:44:12.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.841562 2350 hybrid_clock.cc:648] HybridClock initialized: now 1781919852841490 us; error 85 us; skew 500 ppm
01:44:12.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.842406 2350 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:44:12.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:12 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919852, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:12.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.848671 2350 init.cc:377] Logged in from keytab as kudu/127.2.69.62@KRBTEST.COM (short username kudu)
01:44:12.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.855314 2350 webserver.cc:460] Webserver started at http://127.2.69.62:46827/ using document root <none> and password file <none>
01:44:12.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.856451 2350 fs_manager.cc:359] Metadata directory not provided
01:44:12.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.856699 2350 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:44:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.857229 2350 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:44:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.862206 2350 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data/instance:
01:44:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5e1f9708ea742c2b8a96e91340eaf2b"
01:44:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:12 on dist-test-slave-6rcr"
01:44:12.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.863442 2350 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal/instance:
01:44:12.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5e1f9708ea742c2b8a96e91340eaf2b"
01:44:12.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:12 on dist-test-slave-6rcr"
01:44:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.872186 2350 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
01:44:12.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.878410 2368 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:44:12.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.879698 2350 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
01:44:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.879990 2350 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal
01:44:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5e1f9708ea742c2b8a96e91340eaf2b"
01:44:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:12 on dist-test-slave-6rcr"
01:44:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.880311 2350 fs_report.cc:383] FS layout report
01:44:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:44:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal
01:44:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal
01:44:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data/data
01:44:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:44:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:44:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:44:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:44:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:44:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:44:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:44:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:44:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:44:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:44:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:44:12.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.918511 2350 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:44:12.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.926651 2350 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:44:12.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:12.927227 2350 kserver.cc:163] Server-wide thread pool size limit: 3276
01:44:13.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.036772 2350 rpc_server.cc:307] RPC server started. Bound to: 127.2.69.62:46643
01:44:13.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.036919 2419 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.69.62:46643 every 8 connection(s)
01:44:13.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.039788 2350 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data/info.pb
01:44:13.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.044765 2324 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 2350
01:44:13.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.045342 2324 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal/instance
01:44:13.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.048331 2420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:44:13.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.064983 2420 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } has no permanent_uuid. Determining permanent_uuid...
01:44:13.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.69.61@KRBTEST.COM; defaulting to no policy
01:44:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:44:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.69.61@KRBTEST.COM" created.
01:44:13.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:13 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919852, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.62@KRBTEST.COM for kudu/127.2.69.62@KRBTEST.COM
01:44:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:44:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.69.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.61.keytab.
01:44:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.69.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.61.keytab.
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.105892 2324 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/logs
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data/info.pb
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.61:45953
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.61
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.61.keytab
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.61
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:44:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.119876 2420 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } has no permanent_uuid. Determining permanent_uuid...
01:44:13.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.123445 2370 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:13.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.137040 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.61:45953: Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:13.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.186612 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } attempt: 1
01:44:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.190732 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.61:45953: Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.259176 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } attempt: 2
01:44:13.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.263528 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.61:45953: Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:13.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.355125 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } attempt: 3
01:44:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.359668 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.61:45953: Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.459084 2427 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:44:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.459743 2427 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:44:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.460459 2427 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:44:13.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.493561 2427 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:44:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.493830 2427 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:44:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.494207 2427 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:44:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.494551 2427 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:44:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.494889 2427 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:44:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.495189 2427 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:44:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.495414 2427 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35263/jwks.json
01:44:13.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.503149 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } attempt: 4
01:44:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.507649 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.61:45953: Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.534394 2427 master_runner.cc:386] Master server non-default flags:
01:44:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data
01:44:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.61.keytab
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.61
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.61:45953
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data/info.pb
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.61
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/logs
01:44:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:44:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:44:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:44:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:44:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:44:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:44:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:44:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:44:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:44:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.535806 2427 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:44:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.537101 2427 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:44:13.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.807082 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } attempt: 5
01:44:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.811857 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.61:45953: Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.359277 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } attempt: 6
01:44:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:14.366451 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.61:45953: Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:14.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.556298 2434 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:14.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:13.557751 2427 server_base.cc:1034] running on GCE node
01:44:14.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.556260 2437 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:14.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:13.557531 2435 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:14.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.888665 2427 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:44:14.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.891418 2427 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:44:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.892817 2427 hybrid_clock.cc:648] HybridClock initialized: now 1781919854892783 us; error 47 us; skew 500 ppm
01:44:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.893807 2427 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:44:14.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:14 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919854, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:14.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.903652 2427 init.cc:377] Logged in from keytab as kudu/127.2.69.61@KRBTEST.COM (short username kudu)
01:44:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.909210 2427 webserver.cc:460] Webserver started at http://127.2.69.61:34627/ using document root <none> and password file <none>
01:44:14.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.910240 2427 fs_manager.cc:359] Metadata directory not provided
01:44:14.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.910473 2427 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:44:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.910986 2427 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:44:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.916185 2427 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data/instance:
01:44:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0a5243004414572875e670b63043f72"
01:44:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:14 on dist-test-slave-6rcr"
01:44:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.917271 2427 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal/instance:
01:44:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0a5243004414572875e670b63043f72"
01:44:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:14 on dist-test-slave-6rcr"
01:44:14.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.926162 2427 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
01:44:14.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.934573 2451 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:44:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.935853 2427 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
01:44:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.936168 2427 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal
01:44:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0a5243004414572875e670b63043f72"
01:44:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:14 on dist-test-slave-6rcr"
01:44:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.936480 2427 fs_report.cc:383] FS layout report
01:44:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data/data
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:44:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:44:14.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:44:14.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:44:14.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:44:14.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.983168 2427 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:44:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.991055 2427 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:44:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:14.992303 2427 kserver.cc:163] Server-wide thread pool size limit: 3276
01:44:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.129199 2427 rpc_server.cc:307] RPC server started. Bound to: 127.2.69.61:45953
01:44:15.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.129360 2502 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.69.61:45953 every 8 connection(s)
01:44:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.133627 2427 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data/info.pb
01:44:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.136289 2324 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 2427
01:44:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.136786 2324 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal/instance
01:44:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.140580 2503 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:44:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.69.60@KRBTEST.COM; defaulting to no policy
01:44:15.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.158353 2503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } has no permanent_uuid. Determining permanent_uuid...
01:44:15.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:44:15.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.69.60@KRBTEST.COM" created.
01:44:15.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:15 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919854, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.61@KRBTEST.COM for kudu/127.2.69.62@KRBTEST.COM
01:44:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:44:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.69.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.60.keytab.
01:44:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.69.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.60.keytab.
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.189608 2324 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/logs
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data/info.pb
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:44:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:15.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:15.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.60:38769
01:44:15.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.60
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.60.keytab
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.60
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:44:15.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.200332 2503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } has no permanent_uuid. Determining permanent_uuid...
01:44:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:15 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919854, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.61@KRBTEST.COM for kudu/127.2.69.61@KRBTEST.COM
01:44:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.233764 2503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } has no permanent_uuid. Determining permanent_uuid...
01:44:15.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.236388 2452 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.241060 2503 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.290597 2503 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 1
01:44:15.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.295022 2503 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.363418 2503 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 2
01:44:15.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.367293 2503 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.427098 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } attempt: 7
01:44:15.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:15 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919852, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.62@KRBTEST.COM for kudu/127.2.69.61@KRBTEST.COM
01:44:15.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.449986 2420 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } has no permanent_uuid. Determining permanent_uuid...
01:44:15.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.455981 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.458781 2503 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 3
01:44:15.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.462788 2503 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.514611 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 1
01:44:15.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.518752 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.580106 2509 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:44:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.580657 2509 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:44:15.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.581234 2509 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:44:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.600212 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 2
01:44:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.603991 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.606233 2503 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 4
01:44:15.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.610499 2503 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.616940 2509 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:44:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.617177 2509 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:44:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.617532 2509 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:44:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.617740 2509 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:44:15.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.617929 2509 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:44:15.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.618124 2509 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:44:15.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.618278 2509 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35263/jwks.json
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.656033 2509 master_runner.cc:386] Master server non-default flags:
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.60.keytab
01:44:15.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.60
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.60:38769
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data/info.pb
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.60
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/logs
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:44:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.657486 2509 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:44:15.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.658633 2509 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:44:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.679167 2519 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.689404 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 3
01:44:15.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.693856 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.834270 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 4
01:44:15.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.839984 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:15.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.909884 2503 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 5
01:44:15.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.913940 2503 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:16.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:16.032758 1670 debug-util.cc:398] Leaking SignalData structure 0x7b0800095320 after lost signal to thread 1607
01:44:16.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:16.033478 1670 debug-util.cc:398] Leaking SignalData structure 0x7b0800098660 after lost signal to thread 1673
01:44:16.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.134326 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 5
01:44:16.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:16.139930 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:16.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.461560 2503 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 6
01:44:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:16.470532 2503 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:16.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.692381 2420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 6
01:44:16.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:16.701221 2420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.69.60:38769: Network error: Client connection negotiation failed: client connection to 127.2.69.60:38769: connect: Connection refused (error 111)
01:44:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.679492 2520 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:15.681275 2509 server_base.cc:1034] running on GCE node
01:44:16.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:15.679836 2522 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:16.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.973223 2509 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:44:16.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.976409 2509 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:44:16.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.978001 2509 hybrid_clock.cc:648] HybridClock initialized: now 1781919856977967 us; error 53 us; skew 500 ppm
01:44:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.978992 2509 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:44:16.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:16 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919856, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:16.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.985092 2509 init.cc:377] Logged in from keytab as kudu/127.2.69.60@KRBTEST.COM (short username kudu)
01:44:16.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.991755 2509 webserver.cc:460] Webserver started at http://127.2.69.60:33699/ using document root <none> and password file <none>
01:44:16.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.993115 2509 fs_manager.cc:359] Metadata directory not provided
01:44:16.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.993448 2509 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:44:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:16.994041 2509 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:44:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.001915 2509 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data/instance:
01:44:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bcf9b21db65244a9949da4569500537d"
01:44:17.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:16 on dist-test-slave-6rcr"
01:44:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.003616 2509 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal/instance:
01:44:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bcf9b21db65244a9949da4569500537d"
01:44:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:16 on dist-test-slave-6rcr"
01:44:17.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.014441 2509 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.012s sys 0.000s
01:44:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.021651 2536 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:44:17.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.022997 2509 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
01:44:17.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.023451 2509 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal
01:44:17.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bcf9b21db65244a9949da4569500537d"
01:44:17.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:16 on dist-test-slave-6rcr"
01:44:17.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.023831 2509 fs_report.cc:383] FS layout report
01:44:17.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:44:17.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data/data
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:44:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:44:17.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.069785 2509 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:44:17.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.077993 2509 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:44:17.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.078521 2509 kserver.cc:163] Server-wide thread pool size limit: 3276
01:44:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.204777 2509 rpc_server.cc:307] RPC server started. Bound to: 127.2.69.60:38769
01:44:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.204826 2587 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.69.60:38769 every 8 connection(s)
01:44:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.208864 2509 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data/info.pb
01:44:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.210542 2324 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 2509
01:44:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.211079 2324 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal/instance
01:44:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.223912 2588 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:44:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.253356 2588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } has no permanent_uuid. Determining permanent_uuid...
01:44:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:17 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919851, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.69.60@KRBTEST.COM
01:44:17.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:17 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919856, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.60@KRBTEST.COM for kudu/127.2.69.62@KRBTEST.COM
01:44:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.304369 2588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } has no permanent_uuid. Determining permanent_uuid...
01:44:17.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:17 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919851, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.69.61@KRBTEST.COM
01:44:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:17 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919856, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.60@KRBTEST.COM for kudu/127.2.69.61@KRBTEST.COM
01:44:17.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.351486 2588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } has no permanent_uuid. Determining permanent_uuid...
01:44:17.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:17 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919851, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.69.62@KRBTEST.COM
01:44:17.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:17 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919856, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.60@KRBTEST.COM for kudu/127.2.69.60@KRBTEST.COM
01:44:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.394891 2588 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: Bootstrap starting.
01:44:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.396741 2324 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
01:44:17.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:17.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.403893 2588 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: Neither blocks nor log segments found. Creating new log.
01:44:17.406 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
01:44:17.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.407214 2588 log.cc:824] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: Log is configured to *not* fsync() on all Append() calls
01:44:17.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
01:44:17.422 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
01:44:17.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.424268 2588 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: No bootstrap required, opened a new log
01:44:17.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.69.62"
port: 46643
}
bound_http_address {
host: "127.2.69.62"
port: 46827
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.69.61"
port: 45953
}
bound_http_address {
host: "127.2.69.61"
port: 34627
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.69.60"
port: 38769
}
bound_http_address {
host: "127.2.69.60"
port: 33699
}
}
}
01:44:17.427 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:44:17.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:17.428 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:44:17.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:44:17.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.464984 2588 raft_consensus.cc:348] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:17.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.466275 2588 raft_consensus.cc:374] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:44:17.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.466603 2588 raft_consensus.cc:729] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bcf9b21db65244a9949da4569500537d, State: Initialized, Role: FOLLOWER
01:44:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.467617 2588 consensus_queue.cc:260] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:17 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:44:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.471592 2599 sys_catalog.cc:455] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } } }
01:44:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.473841 2588 sys_catalog.cc:564] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [sys.catalog]: configured and running, proceeding with master startup.
01:44:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.474735 2599 sys_catalog.cc:458] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [sys.catalog]: This master's current role is: FOLLOWER
01:44:17.480 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:17.484 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5cc
01:44:17.495 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:44:17.496 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5cc
01:44:17.499 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:44:17.503 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
01:44:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.517019 2599 raft_consensus.cc:482] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:44:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.517603 2599 raft_consensus.cc:504] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:17.518 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:44:17.519 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:44:17.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.522706 2599 leader_election.cc:288] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643), b0a5243004414572875e670b63043f72 (127.2.69.61:45953)
01:44:17.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.535211 2503 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } attempt: 7
01:44:17.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:17 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:17.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:44:17.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:17 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919854, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.61@KRBTEST.COM for kudu/127.2.69.60@KRBTEST.COM
01:44:17.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:17.584990 2542 tablet.cc:2369] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:44:17.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:17.595 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5cc
01:44:17.597 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:44:17.598 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.69.61:45953
01:44:17.601 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:44:17.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.602620 2324 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu with pid 2427
01:44:17.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:17.625327 2613 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:17.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:17.629335 2613 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:17.647035 2590 negotiation.cc:337] Failed RPC negotiation. Trace:
01:44:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.538216 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:39016 (local address 127.2.69.60:38769)
01:44:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.539299 (+ 1083us) server_negotiation.cc:206] Beginning negotiation
01:44:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.539325 (+ 26us) server_negotiation.cc:399] Waiting for connection header
01:44:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.540257 (+ 932us) server_negotiation.cc:407] Connection header received
01:44:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.541124 (+ 867us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
01:44:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.541158 (+ 34us) server_negotiation.cc:461] Received NEGOTIATE request from client
01:44:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.541481 (+ 323us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
01:44:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.541677 (+ 196us) server_negotiation.cc:226] Negotiated authn=SASL
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.556619 (+ 14942us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.558296 (+ 1677us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.565353 (+ 7057us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.567332 (+ 1979us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.567705 (+ 373us) server_negotiation.cc:657] Negotiated auth-only TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AESGCM(128) Mac=AEAD
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.589098 (+ 21393us) server_negotiation.cc:365] Received SASL_INITIATE NegotiatePB request
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.589148 (+ 50us) server_negotiation.cc:881] Received SASL_INITIATE request from client
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.589194 (+ 46us) server_negotiation.cc:892] Client requested to use mechanism: GSSAPI
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.589430 (+ 236us) server_negotiation.cc:922] calling sasl_server_start()
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.613496 (+ 24066us) server_negotiation.cc:377] Sending SASL_CHALLENGE NegotiatePB response
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 01:44:17.644328 (+ 30832us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:39016: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":612}
01:44:17.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:17.648248 2538 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.2.69.61:45953 (error 108)
01:44:17.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:17.664 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.69.60:38769
01:44:17.665 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:44:17.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.665658 2324 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu with pid 2509
01:44:17.700 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:17.700 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.69.62:46643
01:44:17.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:44:17.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.701431 2324 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu with pid 2350
01:44:17.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:17.727 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.69.61:45953
01:44:17.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:17.731762 2324 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal
01:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data
01:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/logs
01:44:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data/info.pb
01:44:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:44:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:44:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:44:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:44:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.61:45953
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.61
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34627
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.61.keytab
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.61
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:44:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.073678 2616 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:44:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.074378 2616 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:44:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.075105 2616 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:44:18.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.106001 2616 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:44:18.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.106292 2616 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:44:18.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.106685 2616 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:44:18.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.107009 2616 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:44:18.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.107342 2616 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:44:18.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.107656 2616 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:44:18.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.107885 2616 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35263/jwks.json
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:18.149004 2616 master_runner.cc:386] Master server non-default flags:
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.61.keytab
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.61
01:44:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:18.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:18.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:18.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.61:45953
01:44:18.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:18.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:44:18.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:18.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:18.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:18.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:44:18.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data/info.pb
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.61
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34627
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/logs
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:18.150547 2616 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:44:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:18.151899 2616 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:44:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.170898 2623 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.170962 2622 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:18.172426 2616 server_base.cc:1034] running on GCE node
01:44:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:18.170867 2625 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:19.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.407452 2616 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:44:19.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.410145 2616 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:44:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.411547 2616 hybrid_clock.cc:648] HybridClock initialized: now 1781919859411509 us; error 48 us; skew 500 ppm
01:44:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.412444 2616 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:44:19.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:19 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919859, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.418501 2616 init.cc:377] Logged in from keytab as kudu/127.2.69.61@KRBTEST.COM (short username kudu)
01:44:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.425127 2616 webserver.cc:460] Webserver started at http://127.2.69.61:34627/ using document root <none> and password file <none>
01:44:19.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.426102 2616 fs_manager.cc:359] Metadata directory not provided
01:44:19.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.426312 2616 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:44:19.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.434087 2616 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
01:44:19.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.438634 2635 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:44:19.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.439971 2616 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
01:44:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.440275 2616 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal
01:44:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0a5243004414572875e670b63043f72"
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:14 on dist-test-slave-6rcr"
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.442728 2616 fs_report.cc:383] FS layout report
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data/data
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:44:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:44:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:44:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:44:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:44:19.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.492030 2616 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:44:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.498054 2616 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:44:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.498574 2616 kserver.cc:163] Server-wide thread pool size limit: 3276
01:44:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.582783 2616 rpc_server.cc:307] RPC server started. Bound to: 127.2.69.61:45953
01:44:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.582979 2686 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.69.61:45953 every 8 connection(s)
01:44:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.585954 2616 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/data/info.pb
01:44:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.591625 2324 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 2616
01:44:19.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:19.592 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.69.60:38769
01:44:19.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.594662 2324 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/logs
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data/info.pb
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:44:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.60:38769
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.60
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33699
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.60.keytab
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.60
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:44:19.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.599479 2687 sys_catalog.cc:263] Verifying existing consensus state
01:44:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 01:44:19.600452 2687 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:44:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.600930 2616 master.cc:537] Master@127.2.69.61:45953 shutting down...
01:44:19.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:19.672811 2616 master.cc:552] Master@127.2.69.61:45953 shutdown complete.
01:44:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:44:19.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:19.923806 2689 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:44:19.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:19.924610 2689 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:44:19.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:19.925674 2689 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:44:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:19.968518 2689 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:44:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:19.968758 2689 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:44:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:19.969019 2689 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:44:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:19.969216 2689 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:44:19.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:19.969480 2689 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:44:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:19.969794 2689 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:44:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:19.970027 2689 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35263/jwks.json
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:20.006641 2689 master_runner.cc:386] Master server non-default flags:
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.60.keytab
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.60
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.60:38769
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:44:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data/info.pb
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.60
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33699
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/logs
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:44:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:20.008234 2689 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:44:20.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:20.009541 2689 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:44:20.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:20.031895 2696 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:21.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:20.031903 2698 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:20.032672 2689 server_base.cc:1034] running on GCE node
01:44:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:20.032079 2695 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.344877 2689 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:44:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.348141 2689 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:44:21.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.349637 2689 hybrid_clock.cc:648] HybridClock initialized: now 1781919861349602 us; error 56 us; skew 500 ppm
01:44:21.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.350744 2689 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:44:21.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:21 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919861, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:21.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.360813 2689 init.cc:377] Logged in from keytab as kudu/127.2.69.60@KRBTEST.COM (short username kudu)
01:44:21.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.366811 2689 webserver.cc:460] Webserver started at http://127.2.69.60:33699/ using document root <none> and password file <none>
01:44:21.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.367707 2689 fs_manager.cc:359] Metadata directory not provided
01:44:21.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.367902 2689 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:44:21.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.376170 2689 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
01:44:21.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.381062 2707 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:44:21.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.382417 2689 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
01:44:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.382740 2689 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal
01:44:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bcf9b21db65244a9949da4569500537d"
01:44:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:16 on dist-test-slave-6rcr"
01:44:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.385398 2689 fs_report.cc:383] FS layout report
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/wal
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data/data
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:44:21.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:44:21.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.424734 2689 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:44:21.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.432960 2689 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:44:21.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.433624 2689 kserver.cc:163] Server-wide thread pool size limit: 3276
01:44:21.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.509819 2689 rpc_server.cc:307] RPC server started. Bound to: 127.2.69.60:38769
01:44:21.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.510480 2758 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.69.60:38769 every 8 connection(s)
01:44:21.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.513602 2689 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-2/data/info.pb
01:44:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.519865 2324 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 2689
01:44:21.521 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:21.521 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.69.62:46643
01:44:21.521 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:44:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.523016 2324 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu
01:44:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal
01:44:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data
01:44:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:44:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:44:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:44:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/logs
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data/info.pb
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.62:46643
01:44:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.62
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46827
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.62.keytab
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.62
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:44:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.529903 2760 sys_catalog.cc:263] Verifying existing consensus state
01:44:21.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.539907 2760 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: Bootstrap starting.
01:44:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.584619 2760 log.cc:824] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: Log is configured to *not* fsync() on all Append() calls
01:44:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.589931 2760 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:44:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.590786 2760 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: Bootstrap complete.
01:44:21.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.626453 2760 raft_consensus.cc:348] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:21.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.629926 2760 raft_consensus.cc:374] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:44:21.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.630230 2760 raft_consensus.cc:729] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bcf9b21db65244a9949da4569500537d, State: Initialized, Role: FOLLOWER
01:44:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.631183 2760 consensus_queue.cc:260] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:21.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.634193 2764 sys_catalog.cc:455] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } } }
01:44:21.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.635056 2764 sys_catalog.cc:458] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [sys.catalog]: This master's current role is: FOLLOWER
01:44:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.636190 2760 sys_catalog.cc:564] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [sys.catalog]: configured and running, proceeding with master startup.
01:44:21.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.647598 2764 raft_consensus.cc:482] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:44:21.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.648043 2764 raft_consensus.cc:504] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:21.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.653282 2764 leader_election.cc:288] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643), b0a5243004414572875e670b63043f72 (127.2.69.61:45953)
01:44:21.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.667410 2711 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111)
01:44:21.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.682605 2709 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b0a5243004414572875e670b63043f72 (127.2.69.61:45953): Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.684396 2711 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643): Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111)
01:44:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.685874 2711 leader_election.cc:302] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bcf9b21db65244a9949da4569500537d; no voters: a5e1f9708ea742c2b8a96e91340eaf2b, b0a5243004414572875e670b63043f72
01:44:21.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.686822 2764 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:44:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.689072 2713 tablet.cc:2369] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:44:21.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.697525 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:21.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.697885 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.688853 2689 master_runner.cc:186] Error getting master registration for 127.2.69.62:46643: Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111),
01:44:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:21.715943 2689 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:44:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.909415 2762 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:44:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.910174 2762 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:44:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.910887 2762 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:44:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.944206 2762 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:44:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.944511 2762 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:44:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.944988 2762 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:44:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.945348 2762 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:44:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.945808 2762 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:44:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.946134 2762 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:44:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:21.946369 2762 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35263/jwks.json
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:22.009624 2762 master_runner.cc:386] Master server non-default flags:
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.69.20:35123
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.69.62:46643,127.2.69.61:45953,127.2.69.60:38769
01:44:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/krb5kdc/kudu_127.2.69.62.keytab
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.69.62
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.69.62:46643
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35263/jwks.json
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data/info.pb
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.69.62
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46827
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/logs
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:44:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:44:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:44:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:44:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:44:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:44:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:44:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:44:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:44:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:44:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:22.011090 2762 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:44:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:22.012446 2762 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:44:22.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:22.032701 2785 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:22.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:22.704305 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:22.704792 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:23.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:23.619364 2791 raft_consensus.cc:482] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:44:23.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:23.620088 2791 raft_consensus.cc:504] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:23.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:23.651957 2791 leader_election.cc:288] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643), b0a5243004414572875e670b63043f72 (127.2.69.61:45953)
01:44:23.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:23.683065 2711 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643): Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111)
01:44:23.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:23.692876 2709 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b0a5243004414572875e670b63043f72 (127.2.69.61:45953): Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:23.693682 2709 leader_election.cc:302] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bcf9b21db65244a9949da4569500537d; no voters: a5e1f9708ea742c2b8a96e91340eaf2b, b0a5243004414572875e670b63043f72
01:44:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:23.694741 2791 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:44:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:23.709618 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:23.709968 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:24.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:23.431669 2784 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 2762
01:44:24.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:22.033401 2786 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:24.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:24.136391 2788 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:44:24.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:24.131976 2762 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.098s user 0.697s sys 1.161s
01:44:24.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:24.137770 2762 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.104s user 0.697s sys 1.162s
01:44:24.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:24.143643 2787 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2108 milliseconds
01:44:24.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.143687 2762 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:44:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.145561 2762 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:44:24.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.148664 2762 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:44:24.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.150135 2762 hybrid_clock.cc:648] HybridClock initialized: now 1781919864150072 us; error 74 us; skew 500 ppm
01:44:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.151233 2762 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:44:24.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:24 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919864, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:24.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.158879 2762 init.cc:377] Logged in from keytab as kudu/127.2.69.62@KRBTEST.COM (short username kudu)
01:44:24.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.164716 2762 webserver.cc:460] Webserver started at http://127.2.69.62:46827/ using document root <none> and password file <none>
01:44:24.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.165990 2762 fs_manager.cc:359] Metadata directory not provided
01:44:24.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.166285 2762 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:44:24.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.177423 2762 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
01:44:24.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.183818 2800 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:44:24.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.185226 2762 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
01:44:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.185660 2762 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data,/tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal
01:44:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5e1f9708ea742c2b8a96e91340eaf2b"
01:44:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:44:12 on dist-test-slave-6rcr"
01:44:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.189047 2762 fs_report.cc:383] FS layout report
01:44:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:44:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal
01:44:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal
01:44:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data/data
01:44:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:44:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:44:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:44:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:44:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:44:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:44:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:44:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:44:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:44:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:44:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:44:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.239531 2762 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:44:24.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.247601 2762 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:44:24.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.248209 2762 kserver.cc:163] Server-wide thread pool size limit: 3276
01:44:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.335561 2762 rpc_server.cc:307] RPC server started. Bound to: 127.2.69.62:46643
01:44:24.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.335652 2851 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.69.62:46643 every 8 connection(s)
01:44:24.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.339901 2762 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/data/info.pb
01:44:24.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.346393 2324 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu as pid 2762
01:44:24.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:44:24.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.355436 2852 sys_catalog.cc:263] Verifying existing consensus state
01:44:24.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 01:44:24.356534 2852 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:44:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.357460 2762 master.cc:537] Master@127.2.69.62:46643 shutting down...
01:44:24.418 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:24.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:24.464562 2762 master.cc:552] Master@127.2.69.62:46643 shutdown complete.
01:44:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskoRo5Ty/test-tmp/mini-kudu-cluster8354202543082329895/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:44:24.524 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.69.60:38769(127.2.69.60:38769)] connecting to peer
01:44:24.530 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:24.591 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.2.69.60:38769(127.2.69.60:38769)] Successfully connected to peer
01:44:24.596 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:24.597 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:24.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:24.601 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:24.601 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:24.602 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:24.689 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
01:44:24.700 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.69.60
01:44:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:24.713685 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:24.714004 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:24.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:24 dist-test-slave-6rcr krb5kdc[2337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.69.60@KRBTEST.COM
01:44:24.973 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xe57f9d4a, L:/127.0.0.1:46532 - R:/127.2.69.60:38769] using SASL/GSSAPI
01:44:24.990 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:24.998 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:25.002 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:25.005 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.010 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.010 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.011 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.013 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:25.035 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:25.037 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:25.039 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.040 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.040 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.041 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.041 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.042 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.043 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:25.055 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:25.056 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:25.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.059 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.059 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.061 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.062 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:25.076 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:25.078 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:25.080 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.080 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.084 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:25.115 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:25.117 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:25.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.120 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.120 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.121 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.121 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.127 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:25.195 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:25.197 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:25.199 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.199 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.200 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.200 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.202 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:25.215 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:25.217 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:25.220 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.220 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.222 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.222 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.223 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.223 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:25.355 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:25.357 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:25.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.361 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.361 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.362 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.362 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:25.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:25.716943 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:25.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:25.717326 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:25.875 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:25.877 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:25.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.880 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.881 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.881 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.882 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.882 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:25.955 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:25.957 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:25.959 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.959 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.959 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.960 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:25.960 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:25.961 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:26.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:26.718758 2689 master_runner.cc:186] Error getting master registration for 127.2.69.62:46643: Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111),
01:44:26.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:26.719079 2689 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:44:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:26.720659 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:26.720934 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:26.771741 2864 raft_consensus.cc:482] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:44:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:26.772210 2864 raft_consensus.cc:504] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:26.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:26.773949 2864 leader_election.cc:288] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643), b0a5243004414572875e670b63043f72 (127.2.69.61:45953)
01:44:26.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:26.775215 2711 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111) [suppressed 3 similar messages]
01:44:26.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:26.778635 2711 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643): Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111)
01:44:26.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:26.779162 2709 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b0a5243004414572875e670b63043f72 (127.2.69.61:45953): Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:26.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:26.779456 2709 leader_election.cc:302] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bcf9b21db65244a9949da4569500537d; no voters: a5e1f9708ea742c2b8a96e91340eaf2b, b0a5243004414572875e670b63043f72
01:44:26.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:26.779973 2864 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:44:27.075 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:27.076 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:27.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:27.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:27.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:27.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:27.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:27.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:27.080 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:27.435 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:27.436 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:27.439 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:27.439 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:27.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:27.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:27.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:27.440 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:27.441 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:27.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:27.723978 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:27.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:27.724251 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:28.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:28.727139 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:28.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:28.727479 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:29.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:29.730382 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:29.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:29.730655 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:30.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:30.430625 2871 raft_consensus.cc:482] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:44:30.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:30.431282 2871 raft_consensus.cc:504] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:30.433204 2871 leader_election.cc:288] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643), b0a5243004414572875e670b63043f72 (127.2.69.61:45953)
01:44:30.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:30.438858 2711 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643): Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111)
01:44:30.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:30.439438 2709 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b0a5243004414572875e670b63043f72 (127.2.69.61:45953): Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:30.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:30.439806 2709 leader_election.cc:302] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bcf9b21db65244a9949da4569500537d; no voters: a5e1f9708ea742c2b8a96e91340eaf2b, b0a5243004414572875e670b63043f72
01:44:30.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:30.440536 2871 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:44:30.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:30.733898 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:30.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:30.734198 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:31.194 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:31.196 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:31.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:31.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:31.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:31.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:31.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:31.198 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:31.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:31.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:31 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919871, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:31.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:31.365629 2703 init.cc:295] Successfully reacquired a new kerberos TGT
01:44:31.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:31.722179 2689 master_runner.cc:186] Error getting master registration for 127.2.69.62:46643: Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111),
01:44:31.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:31.722602 2689 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:44:31.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:31.736918 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:31.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:31.737268 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:32.275 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:32.276 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:32.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:32.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:32.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:32.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:32.279 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:32.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:32.280 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:32.615 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:32.618 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:32.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:32.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:32.620 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:32.620 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:32.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:32.620 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:32.623 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:32.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:32.740103 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:32.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:32.740358 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:33.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:33.743193 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:33.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:33.743455 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:34.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:34.746340 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:34.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:34.746651 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:34.935 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:34.937 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:34.938 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:34.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:34.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:34.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:34.939 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:34.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:34.940 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:35.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:35.530759 2880 raft_consensus.cc:482] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:44:35.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:35.531113 2880 raft_consensus.cc:504] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:35.532723 2880 leader_election.cc:288] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643), b0a5243004414572875e670b63043f72 (127.2.69.61:45953)
01:44:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:35.534495 2711 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111) [suppressed 3 similar messages]
01:44:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:35.537195 2711 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643): Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111)
01:44:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:35.537637 2709 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b0a5243004414572875e670b63043f72 (127.2.69.61:45953): Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:35.537953 2709 leader_election.cc:302] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bcf9b21db65244a9949da4569500537d; no voters: a5e1f9708ea742c2b8a96e91340eaf2b, b0a5243004414572875e670b63043f72
01:44:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:35.538486 2880 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:44:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:35.749505 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:35.749758 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:36.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:36.725153 2689 master_runner.cc:186] Error getting master registration for 127.2.69.62:46643: Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111),
01:44:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:36.725674 2689 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:44:36.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:36.752926 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:36.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:36.753209 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:36.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:36.837 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:36.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:36.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:36.839 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:36.841 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:36.841 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:36.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:36.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:37.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:37.756258 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:37.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:37.756505 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:37.795 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:37.796 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:37.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:37.798 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:37.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:37.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:37.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:37.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:37.800 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:38.341475 1825 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2180 after lost signal to thread 1756
01:44:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:38.342233 1825 debug-util.cc:398] Leaking SignalData structure 0x7b0800067ae0 after lost signal to thread 1828
01:44:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:38.759425 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:38.759684 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:39.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:39.762465 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:39.762707 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:39.855 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:39.857 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:39.859 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:39.859 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:39.859 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:39.859 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:39.860 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:39.860 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:39.863 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:40.766543 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:40.766873 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:44:41 dist-test-slave-6rcr krb5kdc[2337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781919881, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.69.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:44:41.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:41.368139 2703 init.cc:295] Successfully reacquired a new kerberos TGT
01:44:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:41.728945 2689 master_runner.cc:186] Error getting master registration for 127.2.69.62:46643: Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111),
01:44:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:41.729413 2689 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:44:41.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:41.769759 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:41.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:41.770049 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:41.995 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:41.997 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:41.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:41.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:41.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:41.999 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:42.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:42.000 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:42.001 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:42.772838 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:42.773123 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:43.055 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:43.056 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:43.057 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:43.057 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:43.057 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:43.057 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:43.057 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:43.057 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:43.060 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:43.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:43.776336 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:43.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:43.776681 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:44.135 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:44.136 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:44.137 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:44.137 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:44.137 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:44.137 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:44.137 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:44.137 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:44.141 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:44.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:44.780947 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:44.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:44.781332 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:45.311836 2895 raft_consensus.cc:482] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:44:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:45.312305 2895 raft_consensus.cc:504] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:45.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:45.313930 2895 leader_election.cc:288] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643), b0a5243004414572875e670b63043f72 (127.2.69.61:45953)
01:44:45.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:45.315814 2711 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:44:45.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:45.319504 2711 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643): Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111)
01:44:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:45.320116 2709 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b0a5243004414572875e670b63043f72 (127.2.69.61:45953): Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:45.320397 2709 leader_election.cc:302] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bcf9b21db65244a9949da4569500537d; no voters: a5e1f9708ea742c2b8a96e91340eaf2b, b0a5243004414572875e670b63043f72
01:44:45.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:45.320966 2895 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:44:45.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:45.763095 2168 debug-util.cc:398] Leaking SignalData structure 0x7b08000afc60 after lost signal to thread 2043
01:44:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:45.764309 2168 debug-util.cc:398] Leaking SignalData structure 0x7b0800042080 after lost signal to thread 2171
01:44:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:45.784323 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:45.784590 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:45.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:45.944638 2755 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0dc0 after lost signal to thread 2690
01:44:45.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:45.945143 2755 debug-util.cc:398] Leaking SignalData structure 0x7b0800093820 after lost signal to thread 2758
01:44:46.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:46.764202 2689 master_runner.cc:186] Error getting master registration for 127.2.69.62:46643: Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111),
01:44:46.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:46.764587 2689 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:44:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:46.787545 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:46.787849 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:47.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:47.790647 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:47.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:47.790961 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:47.955 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:47.956 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:47.957 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:47.957 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:47.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:47.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:47.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:47.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:47.960 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:48.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:48.183681 2902 raft_consensus.cc:482] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:44:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:48.184139 2902 raft_consensus.cc:504] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a5e1f9708ea742c2b8a96e91340eaf2b" member_type: VOTER last_known_addr { host: "127.2.69.62" port: 46643 } } peers { permanent_uuid: "b0a5243004414572875e670b63043f72" member_type: VOTER last_known_addr { host: "127.2.69.61" port: 45953 } } peers { permanent_uuid: "bcf9b21db65244a9949da4569500537d" member_type: VOTER last_known_addr { host: "127.2.69.60" port: 38769 } }
01:44:48.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:48.188001 2902 leader_election.cc:288] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643), b0a5243004414572875e670b63043f72 (127.2.69.61:45953)
01:44:48.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:48.191202 2711 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5e1f9708ea742c2b8a96e91340eaf2b (127.2.69.62:46643): Network error: Client connection negotiation failed: client connection to 127.2.69.62:46643: connect: Connection refused (error 111)
01:44:48.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:48.191998 2709 leader_election.cc:334] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b0a5243004414572875e670b63043f72 (127.2.69.61:45953): Network error: Client connection negotiation failed: client connection to 127.2.69.61:45953: connect: Connection refused (error 111)
01:44:48.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:48.192334 2709 leader_election.cc:302] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bcf9b21db65244a9949da4569500537d; no voters: a5e1f9708ea742c2b8a96e91340eaf2b, b0a5243004414572875e670b63043f72
01:44:48.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:48.192960 2902 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:44:48.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:48.793758 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:48.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:48.794015 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:49.796855 2778 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bcf9b21db65244a9949da4569500537d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:44:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:44:49.797132 2778 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:44:50.275 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:50.276 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:50.277 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:50.277 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:50.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:50.277 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:50.277 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:50.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:50.282 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:50.315 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:50.316 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:50.317 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:50.317 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:50.317 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:50.317 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:50.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:50.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:50.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:50.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.61:45953(127.2.69.61:45953)] connecting to peer
01:44:50.756 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.69.62:46643(127.2.69.62:46643)] connecting to peer
01:44:50.757 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:50.757 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:50.757 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.61:45953(127.2.69.61:45953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:50.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.61:45953: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953
01:44:50.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.69.62:46643(127.2.69.62:46643)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:50.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.69.62:46643: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
01:44:50.760 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643; will retry
01:44:50.764 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=26407), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [165ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [239ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [242ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [627ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [632ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [638ms] refreshing cache from master, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [646ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [651ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [654ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [655ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [681ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [682ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [684ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [699ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [701ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [703ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [721ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [721ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [723ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [761ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [762ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [767ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [768ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [835ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [840ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [840ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [842ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [843ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [855ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [861ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [861ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [863ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [864ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [996ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1003ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1515ms] refreshing cache from master, [1516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [1518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [1520ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1524ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1595ms] refreshing cache from master, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [1597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [1598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1601ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [1602ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [1602ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [2715ms] refreshing cache from master, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [2717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [2719ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [2720ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [2721ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [2721ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [3075ms] refreshing cache from master, [3075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [3076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [3077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [3080ms] too many traces: truncated at 100 traces, deferred=Deferred@1617467612(state=PENDING, result=null, callback=(continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556) -> (continuation of Deferred@1628847338 after retry RPC after error@1804014247) -> (continuation of Deferred@1147778988 after retry RPC after error@843142272) -> (continuation of Deferred@1650515790 after retry RPC after error@832695183) -> (continuation of Deferred@956104384 after retry RPC after error@10569739) -> (continuation of Deferred@1683975678 after retry RPC after error@1511792489) -> (continuation of Deferred@97116180 after retry RPC after error@262275510) -> (continuation of Deferred@277720353 after retry RPC after error@718282510) -> (continuation of Deferred@1624474368 after retry RPC after error@2137231645), errback=(continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556) -> (continuation of Deferred@1628847338 after retry RPC after error@1804014247) -> (continuation of Deferred@1147778988 after retry RPC after error@843142272) -> (continuation of Deferred@1650515790 after retry RPC after error@832695183) -> (continuation of Deferred@956104384 after retry RPC after error@10569739) -> (continuation of Deferred@1683975678 after retry RPC after error@1511792489) -> (continuation of Deferred@97116180 after retry RPC after error@262275510) -> (continuation of Deferred@277720353 after retry RPC after error@718282510) -> (continuation of Deferred@1624474368 after retry RPC after error@2137231645)))
01:44:50.771 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.69.62:46643(127.2.69.62:46643)] disconnecting while in state TERMINATED
01:44:50.773 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.69.61:45953(127.2.69.61:45953)] disconnecting while in state TERMINATED
01:44:50.773 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.69.60:38769(127.2.69.60:38769)] disconnecting while in state READY
01:44:50.779 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:44:50.783 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:44:50.785 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.2.69.60:38769(127.2.69.60:38769)] handling channelInactive
01:44:50.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.69.60:38769(127.2.69.60:38769)] cleaning up while in state READY due to: connection closed
01:44:50.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:50.797961 2324 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu with pid 2689
01:44:51.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:44:51Z chronyd exiting
01:44:53.040 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:44:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:53.041276 1594 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu with pid 1874
01:44:53.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:53.068193 1594 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu with pid 2035
01:44:53.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:53.098482 1594 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu with pid 2179
01:44:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:53.125254 1594 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu with pid 1606
01:44:53.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:53.156004 1594 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu with pid 1676
01:44:53.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:44:53.185992 1594 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskoRo5Ty/build/tsan/bin/kudu with pid 1751
01:44:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:44:54Z chronyd exiting
01:44:55.399 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=26407), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [165ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [239ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [242ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [627ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [632ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [638ms] refreshing cache from master, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [646ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [651ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [654ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [655ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [681ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [682ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [684ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [699ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [701ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [703ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [721ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [721ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [723ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [761ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [762ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [767ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [768ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [835ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [840ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [840ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [842ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [843ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [855ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [861ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [861ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [863ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [864ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [996ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1003ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1515ms] refreshing cache from master, [1516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [1518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [1520ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1524ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1595ms] refreshing cache from master, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [1597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [1598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1601ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [1602ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [1602ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [2715ms] refreshing cache from master, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [2717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [2719ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [2720ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [2721ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [2721ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [3075ms] refreshing cache from master, [3075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [3076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [3077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [3080ms] too many traces: truncated at 100 traces, deferred=Deferred@1617467612(state=PENDING, result=null, callback=(continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556) -> (continuation of Deferred@1628847338 after retry RPC after error@1804014247) -> (continuation of Deferred@1147778988 after retry RPC after error@843142272) -> (continuation of Deferred@1650515790 after retry RPC after error@832695183) -> (continuation of Deferred@956104384 after retry RPC after error@10569739) -> (continuation of Deferred@1683975678 after retry RPC after error@1511792489) -> (continuation of Deferred@97116180 after retry RPC after error@262275510) -> (continuation of Deferred@277720353 after retry RPC after error@718282510) -> (continuation of Deferred@1624474368 after retry RPC after error@2137231645), errback=(continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556) -> (continuation of Deferred@1628847338 after retry RPC after error@1804014247) -> (continuation of Deferred@1147778988 after retry RPC after error@843142272) -> (continuation of Deferred@1650515790 after retry RPC after error@832695183) -> (continuation of Deferred@956104384 after retry RPC after error@10569739) -> (continuation of Deferred@1683975678 after retry RPC after error@1511792489) -> (continuation of Deferred@97116180 after retry RPC after error@262275510) -> (continuation of Deferred@277720353 after retry RPC after error@718282510) -> (continuation of Deferred@1624474368 after retry RPC after error@2137231645)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.69.62:46643
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=26407), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [165ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [239ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [242ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [627ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [632ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [638ms] refreshing cache from master, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [646ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [651ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [654ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [655ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [681ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [682ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [684ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [699ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [701ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [703ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [721ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [721ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [723ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [761ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [762ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [767ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [768ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [835ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [840ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [840ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [842ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [843ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [855ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [861ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [861ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [863ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [864ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [996ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1003ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1515ms] refreshing cache from master, [1516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [1518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [1520ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1524ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1595ms] refreshing cache from master, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [1597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [1598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [1601ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [1602ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [1602ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [2715ms] refreshing cache from master, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [2717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [2719ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.61:45953: Network error: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953, [2720ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.62:46643: Network error: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [2721ms] Sub RPC ConnectToMaster: received response from server master-127.2.69.60:38769: OK, [2721ms] delaying RPC due to: Service unavailable: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643, [3075ms] refreshing cache from master, [3075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.61:45953, [3076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.60:38769, [3077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.69.62:46643, [3080ms] too many traces: truncated at 100 traces, deferred=Deferred@1617467612(state=PENDING, result=null, callback=(continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556) -> (continuation of Deferred@1628847338 after retry RPC after error@1804014247) -> (continuation of Deferred@1147778988 after retry RPC after error@843142272) -> (continuation of Deferred@1650515790 after retry RPC after error@832695183) -> (continuation of Deferred@956104384 after retry RPC after error@10569739) -> (continuation of Deferred@1683975678 after retry RPC after error@1511792489) -> (continuation of Deferred@97116180 after retry RPC after error@262275510) -> (continuation of Deferred@277720353 after retry RPC after error@718282510) -> (continuation of Deferred@1624474368 after retry RPC after error@2137231645), errback=(continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556) -> (continuation of Deferred@1628847338 after retry RPC after error@1804014247) -> (continuation of Deferred@1147778988 after retry RPC after error@843142272) -> (continuation of Deferred@1650515790 after retry RPC after error@832695183) -> (continuation of Deferred@956104384 after retry RPC after error@10569739) -> (continuation of Deferred@1683975678 after retry RPC after error@1511792489) -> (continuation of Deferred@97116180 after retry RPC after error@262275510) -> (continuation of Deferred@277720353 after retry RPC after error@718282510) -> (continuation of Deferred@1624474368 after retry RPC after error@2137231645)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.69.61:45953,127.2.69.60:38769,127.2.69.62:46643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.61:45953(127.2.69.61:45953): Connection refused: /127.2.69.61:45953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.69.62:46643(127.2.69.62:46643): Connection refused: /127.2.69.62:46643
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.69.62:46643
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)