20:05:41.387 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:05:41.399 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskEMvaJ5/test-tmp
20:05:41.402 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:05:41.405 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskEMvaJ5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:05:41.618 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:05:41.795 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:05:41.796 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:05:41.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:41.799346 29881 mini_oidc.cc:152] Starting JWKS server
20:05:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:41.803503 29881 webserver.cc:460] Webserver started at http://127.0.0.1:39411/ using document root <none> and password file <none>
20:05:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:41.804026 29881 mini_oidc.cc:189] Starting OIDC Discovery server
20:05:41.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:41.805757 29881 webserver.cc:460] Webserver started at http://127.0.0.1:38873/ using document root <none> and password file <none>
20:05:41.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:05:41Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:05:41.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:05:41Z Disabled control of system clock
20:05:41.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:41.853037 29881 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:41.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:41.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/wal
20:05:41.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/data
20:05:41.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:41.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:41.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:41.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:41.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:41.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:41.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/logs
20:05:41.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/data/info.pb
20:05:41.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:41.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:41.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:41.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:41.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:41.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:41.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:05:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.126:35339
20:05:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.126
20:05:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json with env {}
20:05:42.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.154210 29893 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:05:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.154749 29893 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.155428 29893 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:05:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.196192 29893 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:05:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.196470 29893 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:05:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.196655 29893 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:05:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.196951 29893 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:05:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.197233 29893 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:05:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.197458 29893 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39411/jwks.json
20:05:42.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:42.234169 29893 master_runner.cc:386] Master server non-default flags:
20:05:42.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/data
20:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/wal
20:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.126:35339
20:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/data/info.pb
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.126
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/logs
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:42.235544 29893 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:42.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:42.237617 29893 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:05:42.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.255458 29899 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.255471 29902 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:43.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:42.257948 29893 server_base.cc:1034] running on GCE node
20:05:43.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:42.255529 29900 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:43.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.440673 29893 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:43.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.443353 29893 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:43.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.444694 29893 hybrid_clock.cc:648] HybridClock initialized: now 1790539543444655 us; error 60 us; skew 500 ppm
20:05:43.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.445492 29893 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:43.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.455794 29893 webserver.cc:460] Webserver started at http://127.29.46.126:43161/ using document root <none> and password file <none>
20:05:43.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.456689 29893 fs_manager.cc:359] Metadata directory not provided
20:05:43.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.456872 29893 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:43.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.457280 29893 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:05:43.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.462078 29893 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/data/instance:
20:05:43.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "146eff59d98c4a97a53d4acc90931872"
20:05:43.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:43 on dist-test-slave-3vlk"
20:05:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.463147 29893 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/wal/instance:
20:05:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "146eff59d98c4a97a53d4acc90931872"
20:05:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:43 on dist-test-slave-3vlk"
20:05:43.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.470224 29893 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.003s sys 0.005s
20:05:43.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.475560 29909 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:43.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.476598 29893 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
20:05:43.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.476878 29893 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/wal
20:05:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "146eff59d98c4a97a53d4acc90931872"
20:05:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:43 on dist-test-slave-3vlk"
20:05:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.477161 29893 fs_report.cc:383] FS layout report
20:05:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/wal
20:05:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/wal
20:05:43.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/data/data
20:05:43.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:43.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:43.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:43.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:43.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:43.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:43.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.511363 29893 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:05:43.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.512785 29893 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:43.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.513163 29893 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:43.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.581748 29893 rpc_server.cc:307] RPC server started. Bound to: 127.29.46.126:35339
20:05:43.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.581918 29960 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.46.126:35339 every 8 connection(s)
20:05:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.584499 29893 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/data/info.pb
20:05:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.589522 29961 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:05:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.590341 29881 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 29893
20:05:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.590699 29881 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-0/wal/instance
20:05:43.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.594815 29881 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:43.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:43.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/wal
20:05:43.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/data
20:05:43.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:43.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:43.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:43.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/logs
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/data/info.pb
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:43.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:43.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:43.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:05:43.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:43.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:43.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:43.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.125:36737
20:05:43.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.125
20:05:43.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:43.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:43.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:43.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:43.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:43.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:43.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:43.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json with env {}
20:05:43.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.604094 29961 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } has no permanent_uuid. Determining permanent_uuid...
20:05:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.634863 29961 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } has no permanent_uuid. Determining permanent_uuid...
20:05:43.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.637305 29910 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.46.125:36737: connect: Connection refused (error 111)
20:05:43.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.641947 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.125:36737: Network error: Client connection negotiation failed: client connection to 127.29.46.125:36737: connect: Connection refused (error 111)
20:05:43.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.691442 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } attempt: 1
20:05:43.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.695281 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.125:36737: Network error: Client connection negotiation failed: client connection to 127.29.46.125:36737: connect: Connection refused (error 111)
20:05:43.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.763688 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } attempt: 2
20:05:43.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.767414 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.125:36737: Network error: Client connection negotiation failed: client connection to 127.29.46.125:36737: connect: Connection refused (error 111)
20:05:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.858791 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } attempt: 3
20:05:43.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.862334 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.125:36737: Network error: Client connection negotiation failed: client connection to 127.29.46.125:36737: connect: Connection refused (error 111)
20:05:43.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.900100 29963 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:05:43.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.900645 29963 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:43.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.901237 29963 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:05:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.931989 29963 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:05:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.932263 29963 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:05:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.932448 29963 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:05:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.932623 29963 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:05:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.932807 29963 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:05:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.932996 29963 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39411/jwks.json
20:05:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.967768 29963 master_runner.cc:386] Master server non-default flags:
20:05:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/data
20:05:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/wal
20:05:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:43.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.125:36737
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/data/info.pb
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.125
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:43.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:43.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:43.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:43.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:43.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/logs
20:05:43.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:43.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:43.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:43.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:05:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:05:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:05:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:05:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:05:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.968991 29963 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.971114 29963 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:05:43.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.989705 29972 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:44.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:44.005692 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } attempt: 4
20:05:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:44.009943 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.125:36737: Network error: Client connection negotiation failed: client connection to 127.29.46.125:36737: connect: Connection refused (error 111)
20:05:44.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:44.309315 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } attempt: 5
20:05:44.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:44.313519 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.125:36737: Network error: Client connection negotiation failed: client connection to 127.29.46.125:36737: connect: Connection refused (error 111)
20:05:44.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:44.860997 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } attempt: 6
20:05:44.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:44.868513 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.125:36737: Network error: Client connection negotiation failed: client connection to 127.29.46.125:36737: connect: Connection refused (error 111)
20:05:45.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.989712 29973 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:45.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:43.992168 29963 server_base.cc:1034] running on GCE node
20:05:45.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:43.989827 29975 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:45.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.190560 29963 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:45.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.193758 29963 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:45.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.195233 29963 hybrid_clock.cc:648] HybridClock initialized: now 1790539545195199 us; error 49 us; skew 500 ppm
20:05:45.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.196323 29963 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.204270 29963 webserver.cc:460] Webserver started at http://127.29.46.125:43775/ using document root <none> and password file <none>
20:05:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.205556 29963 fs_manager.cc:359] Metadata directory not provided
20:05:45.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.205843 29963 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:45.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.206413 29963 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:05:45.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.212430 29963 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/data/instance:
20:05:45.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82e74c63ea4e44148d8fb8932be38d4c"
20:05:45.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:45 on dist-test-slave-3vlk"
20:05:45.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.213656 29963 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/wal/instance:
20:05:45.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82e74c63ea4e44148d8fb8932be38d4c"
20:05:45.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:45 on dist-test-slave-3vlk"
20:05:45.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.223109 29963 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
20:05:45.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.230228 29984 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:45.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.231666 29963 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
20:05:45.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.232055 29963 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/wal
20:05:45.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82e74c63ea4e44148d8fb8932be38d4c"
20:05:45.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:45 on dist-test-slave-3vlk"
20:05:45.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.232546 29963 fs_report.cc:383] FS layout report
20:05:45.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:45.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/wal
20:05:45.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/wal
20:05:45.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/data/data
20:05:45.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.290557 29963 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:05:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.291939 29963 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.292332 29963 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:45.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.359865 29963 rpc_server.cc:307] RPC server started. Bound to: 127.29.46.125:36737
20:05:45.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.359958 30035 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.46.125:36737 every 8 connection(s)
20:05:45.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.362381 29963 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/data/info.pb
20:05:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.363451 29881 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 29963
20:05:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.363965 29881 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-1/wal/instance
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.366871 29881 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/wal
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/data
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/logs
20:05:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/data/info.pb
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.124:35859
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.124
20:05:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json with env {}
20:05:45.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.368671 30036 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:05:45.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.380505 30036 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } has no permanent_uuid. Determining permanent_uuid...
20:05:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.403268 30036 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } has no permanent_uuid. Determining permanent_uuid...
20:05:45.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.419947 30036 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } has no permanent_uuid. Determining permanent_uuid...
20:05:45.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.422343 29986 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:45.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.426751 30036 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:45.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.476230 30036 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 1
20:05:45.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.479707 30036 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:45.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.548171 30036 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 2
20:05:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.552340 30036 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:45.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.643805 30036 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 3
20:05:45.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.647512 30036 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:45.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.669487 30038 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:05:45.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.670077 30038 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:45.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.670552 30038 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:05:45.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.701613 30038 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:05:45.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.701936 30038 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:05:45.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.702135 30038 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:05:45.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.702426 30038 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:05:45.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.702732 30038 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:05:45.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.702979 30038 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39411/jwks.json
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.736527 30038 master_runner.cc:386] Master server non-default flags:
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/data
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/wal
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.124:35859
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/data/info.pb
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.124
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/logs
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:05:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:05:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:05:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:05:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:05:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.737982 30038 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:45.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.739825 30038 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:05:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.758831 30049 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:45.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.790928 30036 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 4
20:05:45.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.794423 30036 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.928949 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } attempt: 7
20:05:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:45.956456 29961 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } has no permanent_uuid. Determining permanent_uuid...
20:05:45.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.961694 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:46.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.020133 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 1
20:05:46.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:46.024369 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:46.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.094053 30036 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 5
20:05:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:46.098424 30036 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:46.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.105747 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 2
20:05:46.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:46.109997 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:46.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.195464 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 3
20:05:46.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:46.199787 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.340157 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 4
20:05:46.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:46.344496 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.627869 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 5
20:05:46.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:46.632269 29961 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:46.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.646044 30036 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 6
20:05:46.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:46.652930 30036 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.46.124:35859: Network error: Client connection negotiation failed: client connection to 127.29.46.124:35859: connect: Connection refused (error 111)
20:05:46.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.758985 30048 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:46.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:45.761164 30051 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:46.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:46.965986 30050 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:05:46.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.966138 30038 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:05:46.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.970465 30038 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:46.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.973439 30038 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:46.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.974849 30038 hybrid_clock.cc:648] HybridClock initialized: now 1790539546974793 us; error 73 us; skew 500 ppm
20:05:46.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.975596 30038 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:46.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.981578 30038 webserver.cc:460] Webserver started at http://127.29.46.124:42113/ using document root <none> and password file <none>
20:05:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.982621 30038 fs_manager.cc:359] Metadata directory not provided
20:05:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.982813 30038 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.983242 30038 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:05:46.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.987934 30038 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/data/instance:
20:05:46.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aaece122c36e443e9d09e3cdf9ad284c"
20:05:46.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:46 on dist-test-slave-3vlk"
20:05:46.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.988941 30038 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/wal/instance:
20:05:46.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aaece122c36e443e9d09e3cdf9ad284c"
20:05:46.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:46 on dist-test-slave-3vlk"
20:05:46.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:46.996328 30038 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
20:05:47.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.001644 30063 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:47.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.003592 30038 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:05:47.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.004261 30038 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/wal
20:05:47.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aaece122c36e443e9d09e3cdf9ad284c"
20:05:47.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:46 on dist-test-slave-3vlk"
20:05:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.004748 30038 fs_report.cc:383] FS layout report
20:05:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/wal
20:05:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/wal
20:05:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/data/data
20:05:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:47.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:47.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:47.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:47.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:47.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:47.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:47.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.040148 30038 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:05:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.041705 30038 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.042235 30038 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:47.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.113106 30038 rpc_server.cc:307] RPC server started. Bound to: 127.29.46.124:35859
20:05:47.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.113209 30114 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.46.124:35859 every 8 connection(s)
20:05:47.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.115899 30038 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/data/info.pb
20:05:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.122133 30115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:05:47.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.124722 29881 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 30038
20:05:47.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.125044 29881 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/master-2/wal/instance
20:05:47.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.137099 30115 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } has no permanent_uuid. Determining permanent_uuid...
20:05:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.161401 30115 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } has no permanent_uuid. Determining permanent_uuid...
20:05:47.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.173102 30115 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } has no permanent_uuid. Determining permanent_uuid...
20:05:47.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.184680 29961 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 6
20:05:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.191054 30115 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c: Bootstrap starting.
20:05:47.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.198074 30115 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c: Neither blocks nor log segments found. Creating new log.
20:05:47.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.200709 30115 log.cc:824] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c: Log is configured to *not* fsync() on all Append() calls
20:05:47.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.205574 29961 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872: Bootstrap starting.
20:05:47.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.206007 30115 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c: No bootstrap required, opened a new log
20:05:47.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.212559 29961 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872: Neither blocks nor log segments found. Creating new log.
20:05:47.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.214843 29961 log.cc:824] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872: Log is configured to *not* fsync() on all Append() calls
20:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.219614 29961 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872: No bootstrap required, opened a new log
20:05:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.235730 30115 raft_consensus.cc:348] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } }
20:05:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.236577 30115 raft_consensus.cc:374] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:05:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.236873 30115 raft_consensus.cc:729] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aaece122c36e443e9d09e3cdf9ad284c, State: Initialized, Role: FOLLOWER
20:05:47.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.237716 30115 consensus_queue.cc:260] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } }
20:05:47.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.240801 30126 sys_catalog.cc:455] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } } }
20:05:47.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.242211 30126 sys_catalog.cc:458] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [sys.catalog]: This master's current role is: FOLLOWER
20:05:47.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.242590 30115 sys_catalog.cc:564] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [sys.catalog]: configured and running, proceeding with master startup.
20:05:47.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.252099 29961 raft_consensus.cc:348] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } }
20:05:47.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.252948 29961 raft_consensus.cc:374] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:05:47.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.253234 29961 raft_consensus.cc:729] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 146eff59d98c4a97a53d4acc90931872, State: Initialized, Role: FOLLOWER
20:05:47.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.254347 29961 consensus_queue.cc:260] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } }
20:05:47.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.260213 30134 sys_catalog.cc:455] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } } }
20:05:47.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.261454 30134 sys_catalog.cc:458] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [sys.catalog]: This master's current role is: FOLLOWER
20:05:47.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.262444 29961 sys_catalog.cc:564] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [sys.catalog]: configured and running, proceeding with master startup.
20:05:47.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.264154 30126 raft_consensus.cc:482] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:05:47.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.264678 30126 raft_consensus.cc:504] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } }
20:05:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.269040 29936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aaece122c36e443e9d09e3cdf9ad284c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "146eff59d98c4a97a53d4acc90931872" is_pre_election: true
20:05:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.269075 30126 leader_election.cc:288] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 146eff59d98c4a97a53d4acc90931872 (127.29.46.126:35339), 82e74c63ea4e44148d8fb8932be38d4c (127.29.46.125:36737)
20:05:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.270102 29936 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aaece122c36e443e9d09e3cdf9ad284c in term 0.
20:05:47.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:47.271451 29915 tablet.cc:2369] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:05:47.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.272202 30066 leader_election.cc:302] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 146eff59d98c4a97a53d4acc90931872, aaece122c36e443e9d09e3cdf9ad284c; no voters:
20:05:47.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.282063 30126 raft_consensus.cc:2781] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 0 FOLLOWER]: Leader pre-election won for term 1
20:05:47.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.282972 30126 raft_consensus.cc:482] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:05:47.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.283900 30126 raft_consensus.cc:3037] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 0 FOLLOWER]: Advancing to term 1
20:05:47.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:47.284651 30138 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:05:47.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:47.285107 30138 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:05:47.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.295204 30126 raft_consensus.cc:504] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } }
20:05:47.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:47.297164 30069 tablet.cc:2369] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:05:47.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.301432 30126 leader_election.cc:288] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [CANDIDATE]: Term 1 election: Requested vote from peers 146eff59d98c4a97a53d4acc90931872 (127.29.46.126:35339), 82e74c63ea4e44148d8fb8932be38d4c (127.29.46.125:36737)
20:05:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.302451 29936 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aaece122c36e443e9d09e3cdf9ad284c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "146eff59d98c4a97a53d4acc90931872"
20:05:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.303002 29936 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [term 0 FOLLOWER]: Advancing to term 1
20:05:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.309293 29936 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aaece122c36e443e9d09e3cdf9ad284c in term 1.
20:05:47.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.311570 30066 leader_election.cc:302] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 146eff59d98c4a97a53d4acc90931872, aaece122c36e443e9d09e3cdf9ad284c; no voters:
20:05:47.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.314256 30126 raft_consensus.cc:2781] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 1 FOLLOWER]: Leader election won for term 1
20:05:47.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.272667 30038 master_runner.cc:186] Error getting master registration for 127.29.46.126:35339: OK, instance_id { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" instance_seqno: 1790539543562495 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:05:47.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.318463 30038 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:05:47.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.326427 30126 raft_consensus.cc:686] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [term 1 LEADER]: Becoming Leader. State: Replica: aaece122c36e443e9d09e3cdf9ad284c, State: Running, Role: LEADER
20:05:47.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.327392 30126 consensus_queue.cc:237] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } }
20:05:47.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.338603 30147 sys_catalog.cc:455] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [sys.catalog]: SysCatalogTable state changed. Reason: New leader aaece122c36e443e9d09e3cdf9ad284c. Latest consensus state: current_term: 1 leader_uuid: "aaece122c36e443e9d09e3cdf9ad284c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } } }
20:05:47.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.339490 30147 sys_catalog.cc:458] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [sys.catalog]: This master's current role is: LEADER
20:05:47.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.352679 30152 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:05:47.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.366451 30152 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:05:47.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:47.388818 30153 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:05:47.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:47.389971 30153 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:05:47.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.401926 29936 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [term 1 FOLLOWER]: Refusing update from remote peer aaece122c36e443e9d09e3cdf9ad284c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:05:47.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.405076 30147 consensus_queue.cc:1035] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [LEADER]: Connected to new peer: Peer: permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:05:47.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.355814 29893 master_runner.cc:186] Error getting master registration for 127.29.46.125:36737: OK, instance_id { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" instance_seqno: 1790539545341226 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:05:47.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.415409 29893 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:05:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.444173 30134 sys_catalog.cc:455] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [sys.catalog]: SysCatalogTable state changed. Reason: New leader aaece122c36e443e9d09e3cdf9ad284c. Latest consensus state: current_term: 1 leader_uuid: "aaece122c36e443e9d09e3cdf9ad284c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } } }
20:05:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.444972 30134 sys_catalog.cc:458] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [sys.catalog]: This master's current role is: FOLLOWER
20:05:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.445652 30155 mvcc.cc:205] Tried to move back new op lower bound from 7334049986129121280 to 7334049985863135232. Current Snapshot: MvccSnapshot[applied={T|T < 7334049986129121280}]
20:05:47.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.458323 30156 mvcc.cc:205] Tried to move back new op lower bound from 7334049986129121280 to 7334049985863135232. Current Snapshot: MvccSnapshot[applied={T|T < 7334049986129121280}]
20:05:47.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.474395 30134 sys_catalog.cc:455] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aaece122c36e443e9d09e3cdf9ad284c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } } }
20:05:47.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.475162 30134 sys_catalog.cc:458] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872 [sys.catalog]: This master's current role is: FOLLOWER
20:05:47.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.481564 30147 sys_catalog.cc:455] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aaece122c36e443e9d09e3cdf9ad284c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } } }
20:05:47.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.482362 30147 sys_catalog.cc:458] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [sys.catalog]: This master's current role is: LEADER
20:05:47.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.487152 30152 catalog_manager.cc:1347] Generated new cluster ID: b528ea901d4f4e5b9da68ddd95aa0354
20:05:47.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.487442 30152 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:05:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.550837 30152 catalog_manager.cc:1370] Generated new certificate authority record
20:05:47.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.554344 30152 catalog_manager.cc:1504] Loading token signing keys...
20:05:47.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.597296 30152 catalog_manager.cc:5868] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c: Generated new TSK 0
20:05:47.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.600063 30152 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.632810 29881 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/wal
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/data
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/logs
20:05:47.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/data/info.pb
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.65:0
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.46.65
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.65
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:05:47.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.713443 30036 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } attempt: 7
20:05:47.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.762329 30036 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c: Bootstrap starting.
20:05:47.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.780282 30036 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c: Neither blocks nor log segments found. Creating new log.
20:05:47.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.785746 30036 log.cc:824] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c: Log is configured to *not* fsync() on all Append() calls
20:05:47.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.806340 30036 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c: No bootstrap required, opened a new log
20:05:47.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.853901 30036 raft_consensus.cc:348] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } }
20:05:47.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.854744 30036 raft_consensus.cc:374] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:05:47.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.855094 30036 raft_consensus.cc:729] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82e74c63ea4e44148d8fb8932be38d4c, State: Initialized, Role: FOLLOWER
20:05:47.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.856087 30036 consensus_queue.cc:260] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } }
20:05:47.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.864989 30164 sys_catalog.cc:455] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } } }
20:05:47.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.865734 30164 sys_catalog.cc:458] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [sys.catalog]: This master's current role is: FOLLOWER
20:05:47.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.868443 30036 sys_catalog.cc:564] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [sys.catalog]: configured and running, proceeding with master startup.
20:05:47.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.868993 30009 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [term 0 FOLLOWER]: Advancing to term 1
20:05:47.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.874012 30011 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aaece122c36e443e9d09e3cdf9ad284c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82e74c63ea4e44148d8fb8932be38d4c" is_pre_election: true
20:05:47.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.875494 30010 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aaece122c36e443e9d09e3cdf9ad284c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82e74c63ea4e44148d8fb8932be38d4c"
20:05:47.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.878556 30009 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [term 1 FOLLOWER]: Refusing update from remote peer aaece122c36e443e9d09e3cdf9ad284c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:05:47.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.880446 30126 consensus_queue.cc:1035] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [LEADER]: Connected to new peer: Peer: permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:05:47.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.957469 30164 sys_catalog.cc:455] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [sys.catalog]: SysCatalogTable state changed. Reason: New leader aaece122c36e443e9d09e3cdf9ad284c. Latest consensus state: current_term: 1 leader_uuid: "aaece122c36e443e9d09e3cdf9ad284c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } } }
20:05:47.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.958246 30164 sys_catalog.cc:458] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [sys.catalog]: This master's current role is: FOLLOWER
20:05:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.961694 30175 sys_catalog.cc:455] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aaece122c36e443e9d09e3cdf9ad284c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } } }
20:05:47.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.963977 30175 sys_catalog.cc:458] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c [sys.catalog]: This master's current role is: FOLLOWER
20:05:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:47.972612 30180 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:05:48.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:48.134986 30160 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:05:48.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:48.135596 30160 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:48.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:48.136250 30160 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:05:48.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:48.191938 30160 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:05:48.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:48.193329 30160 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.46.65
20:05:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.254094 30160 tablet_server_runner.cc:78] Tablet server non-default flags:
20:05:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/data
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/wal
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.65:0
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/data/info.pb
20:05:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.65
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.46.65
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/logs
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:05:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:48.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.255627 30160 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:48.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.257632 30160 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:05:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:48.280970 30186 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:48.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.290004 30126 sys_catalog.cc:455] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aaece122c36e443e9d09e3cdf9ad284c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "146eff59d98c4a97a53d4acc90931872" member_type: VOTER last_known_addr { host: "127.29.46.126" port: 35339 } } peers { permanent_uuid: "82e74c63ea4e44148d8fb8932be38d4c" member_type: VOTER last_known_addr { host: "127.29.46.125" port: 36737 } } peers { permanent_uuid: "aaece122c36e443e9d09e3cdf9ad284c" member_type: VOTER last_known_addr { host: "127.29.46.124" port: 35859 } } }
20:05:48.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.291409 30126 sys_catalog.cc:458] T 00000000000000000000000000000000 P aaece122c36e443e9d09e3cdf9ad284c [sys.catalog]: This master's current role is: LEADER
20:05:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.397816 30153 catalog_manager.cc:1259] Loaded cluster ID: b528ea901d4f4e5b9da68ddd95aa0354
20:05:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.398161 30153 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872: loading cluster ID for follower catalog manager: success
20:05:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.420109 30153 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872: acquiring CA information for follower catalog manager: success
20:05:48.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.439541 30153 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 146eff59d98c4a97a53d4acc90931872: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:05:48.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.993887 30180 catalog_manager.cc:1259] Loaded cluster ID: b528ea901d4f4e5b9da68ddd95aa0354
20:05:48.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:48.994557 30180 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c: loading cluster ID for follower catalog manager: success
20:05:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:49.017966 30180 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c: acquiring CA information for follower catalog manager: success
20:05:49.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:49.033027 30180 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 82e74c63ea4e44148d8fb8932be38d4c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:05:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:50.006388 30160 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.724s user 0.511s sys 0.760s
20:05:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:50.007294 30188 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:05:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:50.007866 30160 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.726s user 0.511s sys 0.760s
20:05:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:49.673911 30185 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 30160
20:05:50.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:50.009547 30189 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:50.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:48.286824 30187 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.013062 30160 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:05:50.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.016487 30160 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.030299 30160 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:50.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.034186 30160 hybrid_clock.cc:648] HybridClock initialized: now 1790539550034106 us; error 83 us; skew 500 ppm
20:05:50.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.035478 30160 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.078485 30160 webserver.cc:460] Webserver started at http://127.29.46.65:36099/ using document root <none> and password file <none>
20:05:50.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.080111 30160 fs_manager.cc:359] Metadata directory not provided
20:05:50.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.080629 30160 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:50.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.081326 30160 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:05:50.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.088485 30160 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/data/instance:
20:05:50.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04532465b505417f825defbef4337bc5"
20:05:50.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:50 on dist-test-slave-3vlk"
20:05:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.090240 30160 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/wal/instance:
20:05:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04532465b505417f825defbef4337bc5"
20:05:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:50 on dist-test-slave-3vlk"
20:05:50.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.118608 30160 fs_manager.cc:691] Time spent creating directory manager: real 0.027s user 0.009s sys 0.017s
20:05:50.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.129571 30202 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.131026 30160 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.002s sys 0.002s
20:05:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.131605 30160 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/wal
20:05:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04532465b505417f825defbef4337bc5"
20:05:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:50 on dist-test-slave-3vlk"
20:05:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.132256 30160 fs_report.cc:383] FS layout report
20:05:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/wal
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/wal
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/data/data
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:50.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.233641 30160 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:05:50.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.236310 30160 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:50.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.237006 30160 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:50.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.246626 30160 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:05:50.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.258647 30160 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:05:50.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.259119 30160 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:05:50.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.259582 30160 ts_tablet_manager.cc:610] Registered 0 tablets
20:05:50.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.259923 30160 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:05:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.804932 30160 rpc_server.cc:307] RPC server started. Bound to: 127.29.46.65:46843
20:05:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.805066 30314 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.46.65:46843 every 8 connection(s)
20:05:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.808223 30160 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/data/info.pb
20:05:50.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.816769 29881 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 30160
20:05:50.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.817329 29881 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-0/wal/instance
20:05:50.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.828958 29881 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:50.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:50.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/wal
20:05:50.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/data
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/logs
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/data/info.pb
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:50.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:05:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.66:0
20:05:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.46.66
20:05:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.66
20:05:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:05:50.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.875209 30316 heartbeater.cc:347] Connected to a master server at 127.29.46.125:36737
20:05:50.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.875730 30316 heartbeater.cc:464] Registering TS with master...
20:05:50.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.876914 30316 heartbeater.cc:511] Master 127.29.46.125:36737 requested a full tablet report, sending...
20:05:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.880005 30315 heartbeater.cc:347] Connected to a master server at 127.29.46.124:35859
20:05:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.880336 30315 heartbeater.cc:464] Registering TS with master...
20:05:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.881135 30315 heartbeater.cc:511] Master 127.29.46.124:35859 requested a full tablet report, sending...
20:05:50.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.881578 30000 ts_manager.cc:194] Registered new tserver with Master: 04532465b505417f825defbef4337bc5 (127.29.46.65:46843)
20:05:50.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.882066 30317 heartbeater.cc:347] Connected to a master server at 127.29.46.126:35339
20:05:50.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.882376 30317 heartbeater.cc:464] Registering TS with master...
20:05:50.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.883152 30317 heartbeater.cc:511] Master 127.29.46.126:35339 requested a full tablet report, sending...
20:05:50.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.885324 30080 ts_manager.cc:194] Registered new tserver with Master: 04532465b505417f825defbef4337bc5 (127.29.46.65:46843)
20:05:50.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.885901 29926 ts_manager.cc:194] Registered new tserver with Master: 04532465b505417f825defbef4337bc5 (127.29.46.65:46843)
20:05:50.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:50.887619 30080 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.46.65:53043
20:05:51.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:51.188032 30325 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:05:51.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:51.188592 30325 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:51.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:51.189217 30325 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:05:51.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:51.220661 30325 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:05:51.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:51.221623 30325 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.46.66
20:05:51.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:51.255523 30325 tablet_server_runner.cc:78] Tablet server non-default flags:
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/data
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/wal
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.66:0
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/data/info.pb
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.66
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:51.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:51.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:51.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:51.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:51.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.46.66
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/logs
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:51.257159 30325 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:51.259081 30325 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:05:51.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:51.282701 30332 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:51.891435 30315 heartbeater.cc:503] Master 127.29.46.124:35859 was elected leader, sending a full tablet report...
20:05:52.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:51.286623 30334 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:52.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:51.282750 30331 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:52.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:52.657531 30333 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1371 milliseconds
20:05:52.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.657629 30325 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:05:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.658931 30325 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.661103 30325 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:52.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.662467 30325 hybrid_clock.cc:648] HybridClock initialized: now 1790539552662413 us; error 53 us; skew 500 ppm
20:05:52.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.663305 30325 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:52.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.669169 30325 webserver.cc:460] Webserver started at http://127.29.46.66:46587/ using document root <none> and password file <none>
20:05:52.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.670148 30325 fs_manager.cc:359] Metadata directory not provided
20:05:52.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.670373 30325 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.670866 30325 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:05:52.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.675617 30325 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/data/instance:
20:05:52.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "080e618b33fd4702a52bade615214778"
20:05:52.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:52 on dist-test-slave-3vlk"
20:05:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.676739 30325 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/wal/instance:
20:05:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "080e618b33fd4702a52bade615214778"
20:05:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:52 on dist-test-slave-3vlk"
20:05:52.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.683840 30325 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
20:05:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.689697 30344 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:52.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.690891 30325 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:05:52.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.691216 30325 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/wal
20:05:52.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "080e618b33fd4702a52bade615214778"
20:05:52.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:52 on dist-test-slave-3vlk"
20:05:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.691550 30325 fs_report.cc:383] FS layout report
20:05:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/wal
20:05:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/wal
20:05:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/data/data
20:05:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:52.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.737488 30325 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:05:52.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.739161 30325 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:52.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.739661 30325 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.742220 30325 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:05:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.746356 30325 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:05:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.746572 30325 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:05:52.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.746882 30325 ts_tablet_manager.cc:610] Registered 0 tablets
20:05:52.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.747057 30325 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:05:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.888942 30325 rpc_server.cc:307] RPC server started. Bound to: 127.29.46.66:35579
20:05:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.889096 30456 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.46.66:35579 every 8 connection(s)
20:05:52.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.891734 30325 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/data/info.pb
20:05:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.893146 29881 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 30325
20:05:52.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.893900 29881 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-1/wal/instance
20:05:52.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.912477 29881 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/wal
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/data
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/logs
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/data/info.pb
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.67:0
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.46.67
20:05:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.67
20:05:52.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:52.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:52.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:52.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:52.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:52.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:05:52.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.936599 30458 heartbeater.cc:347] Connected to a master server at 127.29.46.125:36737
20:05:52.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.937036 30458 heartbeater.cc:464] Registering TS with master...
20:05:52.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.938232 30458 heartbeater.cc:511] Master 127.29.46.125:36737 requested a full tablet report, sending...
20:05:52.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.941182 30000 ts_manager.cc:194] Registered new tserver with Master: 080e618b33fd4702a52bade615214778 (127.29.46.66:35579)
20:05:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.947417 30457 heartbeater.cc:347] Connected to a master server at 127.29.46.124:35859
20:05:52.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.947758 30457 heartbeater.cc:464] Registering TS with master...
20:05:52.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.948524 30457 heartbeater.cc:511] Master 127.29.46.124:35859 requested a full tablet report, sending...
20:05:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.948917 30461 heartbeater.cc:347] Connected to a master server at 127.29.46.126:35339
20:05:52.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.949227 30461 heartbeater.cc:464] Registering TS with master...
20:05:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.949988 30461 heartbeater.cc:511] Master 127.29.46.126:35339 requested a full tablet report, sending...
20:05:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.950623 30080 ts_manager.cc:194] Registered new tserver with Master: 080e618b33fd4702a52bade615214778 (127.29.46.66:35579)
20:05:52.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.952308 30080 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.46.66:42717
20:05:52.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:52.952562 29926 ts_manager.cc:194] Registered new tserver with Master: 080e618b33fd4702a52bade615214778 (127.29.46.66:35579)
20:05:53.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:53.226392 30467 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:05:53.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:53.226831 30467 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:53.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:53.227237 30467 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:05:53.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:53.258009 30467 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:05:53.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:53.258931 30467 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.46.67
20:05:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:53.293239 30467 tablet_server_runner.cc:78] Tablet server non-default flags:
20:05:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.46.84:46311
20:05:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/data
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/wal
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.46.67:0
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/data/info.pb
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.46.67
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.46.126:35339,127.29.46.125:36737,127.29.46.124:35859
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.46.67
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/logs
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:05:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:53.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:53.294768 30467 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:53.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:53.296484 30467 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:05:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:53.314268 30473 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:53.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:53.956558 30457 heartbeater.cc:503] Master 127.29.46.124:35859 was elected leader, sending a full tablet report...
20:05:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:53.314313 30476 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:53.316540 30474 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:54.498440 30475 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:05:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.498453 30467 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:05:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.503540 30467 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.506192 30467 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.507615 30467 hybrid_clock.cc:648] HybridClock initialized: now 1790539554507578 us; error 57 us; skew 500 ppm
20:05:54.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.508351 30467 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:54.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.515064 30467 webserver.cc:460] Webserver started at http://127.29.46.67:35181/ using document root <none> and password file <none>
20:05:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.516001 30467 fs_manager.cc:359] Metadata directory not provided
20:05:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.516192 30467 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.516670 30467 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:05:54.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.521341 30467 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/data/instance:
20:05:54.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eab82d4e8abf4d2abc64e560adfa413f"
20:05:54.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:54 on dist-test-slave-3vlk"
20:05:54.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.522487 30467 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/wal/instance:
20:05:54.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eab82d4e8abf4d2abc64e560adfa413f"
20:05:54.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:54 on dist-test-slave-3vlk"
20:05:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.529263 30467 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.002s sys 0.005s
20:05:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.534519 30484 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:54.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.535521 30467 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
20:05:54.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.535879 30467 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/wal
20:05:54.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eab82d4e8abf4d2abc64e560adfa413f"
20:05:54.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:54 on dist-test-slave-3vlk"
20:05:54.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.536237 30467 fs_report.cc:383] FS layout report
20:05:54.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:54.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/wal
20:05:54.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/wal
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/data/data
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:54.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.580682 30467 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:05:54.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.582214 30467 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:54.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.582708 30467 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:54.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.585258 30467 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:05:54.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.589283 30467 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:05:54.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.589514 30467 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:05:54.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.589838 30467 ts_tablet_manager.cc:610] Registered 0 tablets
20:05:54.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.590026 30467 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:05:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.725026 30467 rpc_server.cc:307] RPC server started. Bound to: 127.29.46.67:33417
20:05:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.725122 30596 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.46.67:33417 every 8 connection(s)
20:05:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.727650 30467 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/data/info.pb
20:05:54.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.733085 29881 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 30467
20:05:54.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.733613 29881 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster619456843172718188/ts-2/wal/instance
20:05:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.775187 30598 heartbeater.cc:347] Connected to a master server at 127.29.46.125:36737
20:05:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.775655 30598 heartbeater.cc:464] Registering TS with master...
20:05:54.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.776737 30598 heartbeater.cc:511] Master 127.29.46.125:36737 requested a full tablet report, sending...
20:05:54.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.779708 30597 heartbeater.cc:347] Connected to a master server at 127.29.46.124:35859
20:05:54.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.779750 30000 ts_manager.cc:194] Registered new tserver with Master: eab82d4e8abf4d2abc64e560adfa413f (127.29.46.67:33417)
20:05:54.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.780030 30597 heartbeater.cc:464] Registering TS with master...
20:05:54.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.780701 30597 heartbeater.cc:511] Master 127.29.46.124:35859 requested a full tablet report, sending...
20:05:54.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.781652 30599 heartbeater.cc:347] Connected to a master server at 127.29.46.126:35339
20:05:54.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.781996 30599 heartbeater.cc:464] Registering TS with master...
20:05:54.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.782621 30079 ts_manager.cc:194] Registered new tserver with Master: eab82d4e8abf4d2abc64e560adfa413f (127.29.46.67:33417)
20:05:54.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.782773 30599 heartbeater.cc:511] Master 127.29.46.126:35339 requested a full tablet report, sending...
20:05:54.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.784536 30079 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.46.67:33847
20:05:54.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.785072 29925 ts_manager.cc:194] Registered new tserver with Master: eab82d4e8abf4d2abc64e560adfa413f (127.29.46.67:33417)
20:05:54.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:54.801226 29881 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:05:54.815 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:05:54.816 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:05:54.827 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.29.46.126"
port: 35339
}
bound_http_address {
host: "127.29.46.126"
port: 43161
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.29.46.125"
port: 36737
}
bound_http_address {
host: "127.29.46.125"
port: 43775
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.29.46.124"
port: 35859
}
bound_http_address {
host: "127.29.46.124"
port: 42113
}
}
}
20:05:54.831 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:05:54.834 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.29.46.65"
port: 46843
}
bound_http_address {
host: "127.29.46.65"
port: 36099
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.29.46.66"
port: 35579
}
bound_http_address {
host: "127.29.46.66"
port: 46587
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.29.46.67"
port: 33417
}
bound_http_address {
host: "127.29.46.67"
port: 35181
}
}
}
20:05:54.834 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:05:55.007 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:05:55.117 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskEMvaJ5/test-tmp
20:05:55.118 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:05:55.118 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskEMvaJ5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:05:55.122 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:05:55.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:05:55.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:05:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:55.466554 30608 mini_oidc.cc:152] Starting JWKS server
20:05:55.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:55.470337 30608 webserver.cc:460] Webserver started at http://127.0.0.1:38181/ using document root <none> and password file <none>
20:05:55.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:55.470824 30608 mini_oidc.cc:189] Starting OIDC Discovery server
20:05:55.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:55.472494 30608 webserver.cc:460] Webserver started at http://127.0.0.1:39463/ using document root <none> and password file <none>
20:05:55.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:05:55.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/principal' for realm 'KRBTEST.COM',
20:05:55.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:05:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:05:55 dist-test-slave-3vlk krb5kdc[30620](info): setting up network...
20:05:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:05:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:05:55 dist-test-slave-3vlk krb5kdc[30620](info): set up 2 sockets
20:05:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:05:55 dist-test-slave-3vlk krb5kdc[30620](info): commencing operation
20:05:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:05:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:55.787735 30597 heartbeater.cc:503] Master 127.29.46.124:35859 was elected leader, sending a full tablet report...
20:05:56.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:56.574225 30608 mini_kdc.cc:121] Time spent starting KDC: real 1.078s user 0.000s sys 0.018s
20:05:56.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:05:56.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:05:56.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:05:56.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:05:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:05:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:05:56.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:05:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:05:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:05:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:05:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/test-user.keytab.
20:05:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/test-user.keytab.
20:05:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:05:56 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:05:56.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:05:56Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:05:56.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:05:56Z Disabled control of system clock
20:05:56.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.228.62@KRBTEST.COM; defaulting to no policy
20:05:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:05:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.228.62@KRBTEST.COM" created.
20:05:56.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:05:56.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.228.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.62.keytab.
20:05:56.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.228.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.62.keytab.
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:56.753867 30608 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/logs
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data/info.pb
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:56.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.62:44523
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.62
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.62.keytab
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.62
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:05:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:05:57.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.054131 30634 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:05:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.054792 30634 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:57.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.055552 30634 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:05:57.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.086295 30634 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:05:57.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.086563 30634 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:05:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.086928 30634 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:05:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.087250 30634 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:05:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.087579 30634 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:05:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.087791 30634 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:05:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.087978 30634 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38181/jwks.json
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:57.124641 30634 master_runner.cc:386] Master server non-default flags:
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.62.keytab
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.62
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.62:44523
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data/info.pb
20:05:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.62
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/logs
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:57.125706 30634 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:57.126936 30634 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:05:57.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.146349 30640 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:58.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.146443 30643 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:58.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:57.146416 30641 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:05:58.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.390383 30642 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1243 milliseconds
20:05:58.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.390501 30634 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:05:58.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.391726 30634 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:05:58.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.394762 30634 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:05:58.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.396220 30634 hybrid_clock.cc:648] HybridClock initialized: now 1790539558396150 us; error 78 us; skew 500 ppm
20:05:58.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.397022 30634 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:05:58.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:05:58 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539558, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:05:58.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.402705 30634 init.cc:377] Logged in from keytab as kudu/127.29.228.62@KRBTEST.COM (short username kudu)
20:05:58.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.408401 30634 webserver.cc:460] Webserver started at http://127.29.228.62:44779/ using document root <none> and password file <none>
20:05:58.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.409237 30634 fs_manager.cc:359] Metadata directory not provided
20:05:58.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.409452 30634 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:05:58.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.409899 30634 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:05:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.414530 30634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data/instance:
20:05:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e67406bc78847358dbeda373a66940e"
20:05:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:58 on dist-test-slave-3vlk"
20:05:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.415522 30634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal/instance:
20:05:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e67406bc78847358dbeda373a66940e"
20:05:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:58 on dist-test-slave-3vlk"
20:05:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.423090 30634 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
20:05:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.428483 30652 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:05:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.429499 30634 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.429831 30634 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e67406bc78847358dbeda373a66940e"
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:58 on dist-test-slave-3vlk"
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.430132 30634 fs_report.cc:383] FS layout report
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data/data
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:05:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:05:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.468022 30634 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:05:58.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.473376 30634 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:05:58.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.473762 30634 kserver.cc:163] Server-wide thread pool size limit: 3276
20:05:58.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.543361 30634 rpc_server.cc:307] RPC server started. Bound to: 127.29.228.62:44523
20:05:58.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.543442 30703 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.228.62:44523 every 8 connection(s)
20:05:58.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.546010 30634 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data/info.pb
20:05:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.551242 30704 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:05:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.555313 30608 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 30634
20:05:58.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.555756 30608 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal/instance
20:05:58.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.562939 30704 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } has no permanent_uuid. Determining permanent_uuid...
20:05:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.228.61@KRBTEST.COM; defaulting to no policy
20:05:58.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:05:58.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.228.61@KRBTEST.COM" created.
20:05:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:05:58 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539558, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.62@KRBTEST.COM for kudu/127.29.228.62@KRBTEST.COM
20:05:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:05:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.228.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.61.keytab.
20:05:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.228.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.61.keytab.
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.601956 30608 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/logs
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data/info.pb
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.61:33285
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.61
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.61.keytab
20:05:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.61
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.606503 30704 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } has no permanent_uuid. Determining permanent_uuid...
20:05:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.608963 30653 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:05:58.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.616416 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.61:33285: Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:05:58.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.665894 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } attempt: 1
20:05:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.669317 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.61:33285: Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:05:58.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.737736 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } attempt: 2
20:05:58.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.741183 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.61:33285: Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:05:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.832535 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } attempt: 3
20:05:58.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.835942 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.61:33285: Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:05:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.908476 30710 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:05:58.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.909052 30710 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:05:58.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.909703 30710 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:05:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.942946 30710 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:05:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.943221 30710 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:05:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.943490 30710 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:05:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.943682 30710 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:05:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.943874 30710 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:05:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.944057 30710 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:05:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.944272 30710 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38181/jwks.json
20:05:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.979367 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } attempt: 4
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:58.982735 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.61:33285: Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.983290 30710 master_runner.cc:386] Master server non-default flags:
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.61.keytab
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.61
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.61:33285
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:05:58.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data/info.pb
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.61
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/logs
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:05:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.984346 30710 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:05:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:58.985342 30710 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:05:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:59.282135 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } attempt: 5
20:05:59.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:59.286304 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.61:33285: Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:05:59.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:59.833658 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } attempt: 6
20:05:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:59.839955 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.61:33285: Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:06:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:59.001729 30718 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:59.001734 30717 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:05:59.001736 30720 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:05:59.003532 30710 server_base.cc:1034] running on GCE node
20:06:00.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.231194 30710 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:06:00.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.234128 30710 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:06:00.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.235528 30710 hybrid_clock.cc:648] HybridClock initialized: now 1790539560235485 us; error 57 us; skew 500 ppm
20:06:00.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.236265 30710 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:06:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:00 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539560, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:06:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.241942 30710 init.cc:377] Logged in from keytab as kudu/127.29.228.61@KRBTEST.COM (short username kudu)
20:06:00.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.247370 30710 webserver.cc:460] Webserver started at http://127.29.228.61:38609/ using document root <none> and password file <none>
20:06:00.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.248255 30710 fs_manager.cc:359] Metadata directory not provided
20:06:00.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.248526 30710 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:06:00.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.248971 30710 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.254303 30710 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data/instance:
20:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2680642a507e428b8b566bbc8bb47c24"
20:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:06:00 on dist-test-slave-3vlk"
20:06:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.255491 30710 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal/instance:
20:06:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2680642a507e428b8b566bbc8bb47c24"
20:06:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:06:00 on dist-test-slave-3vlk"
20:06:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.263211 30710 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
20:06:00.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.268683 30730 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:06:00.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.270009 30710 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:06:00.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.270345 30710 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal
20:06:00.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2680642a507e428b8b566bbc8bb47c24"
20:06:00.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:06:00 on dist-test-slave-3vlk"
20:06:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.270751 30710 fs_report.cc:383] FS layout report
20:06:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:06:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal
20:06:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal
20:06:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data/data
20:06:00.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:06:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:06:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:06:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:06:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:06:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:06:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:06:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:06:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:06:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:06:00.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:06:00.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.334048 30710 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:06:00.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.339589 30710 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:06:00.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.340113 30710 kserver.cc:163] Server-wide thread pool size limit: 3276
20:06:00.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.420771 30710 rpc_server.cc:307] RPC server started. Bound to: 127.29.228.61:33285
20:06:00.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.420929 30782 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.228.61:33285 every 8 connection(s)
20:06:00.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.423523 30710 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data/info.pb
20:06:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.429332 30783 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:06:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.431540 30608 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 30710
20:06:00.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.431990 30608 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal/instance
20:06:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.444069 30783 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } has no permanent_uuid. Determining permanent_uuid...
20:06:00.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.228.60@KRBTEST.COM; defaulting to no policy
20:06:00.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:06:00.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.228.60@KRBTEST.COM" created.
20:06:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:00 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539560, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.61@KRBTEST.COM for kudu/127.29.228.62@KRBTEST.COM
20:06:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:06:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.228.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.60.keytab.
20:06:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.228.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.60.keytab.
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.478847 30608 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/logs
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data/info.pb
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.60:38133
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.60
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:06:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.60.keytab
20:06:00.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.60
20:06:00.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:06:00.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:06:00.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:06:00.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:06:00.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:00.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:06:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:06:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:06:00.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.484696 30783 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } has no permanent_uuid. Determining permanent_uuid...
20:06:00.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:00 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539560, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.61@KRBTEST.COM for kudu/127.29.228.61@KRBTEST.COM
20:06:00.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.511967 30783 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } has no permanent_uuid. Determining permanent_uuid...
20:06:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.514081 30732 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.518201 30783 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:00.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.567651 30783 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 1
20:06:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.571771 30783 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.640153 30783 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 2
20:06:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.644286 30783 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.735697 30783 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 3
20:06:00.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.739044 30783 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:00.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.796046 30789 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:06:00.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.796650 30789 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:06:00.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.797230 30789 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:06:00.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.831316 30789 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:06:00.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.831537 30789 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:06:00.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.831796 30789 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:06:00.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.831986 30789 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:06:00.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.832171 30789 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:06:00.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.832351 30789 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:06:00.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.832486 30789 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38181/jwks.json
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.870752 30789 master_runner.cc:386] Master server non-default flags:
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.60.keytab
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.60
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.60:38133
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data/info.pb
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.60
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/logs
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.871845 30789 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.872743 30789 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:06:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.882475 30783 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 4
20:06:00.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.886925 30783 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.891379 30797 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:00.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.900552 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } attempt: 7
20:06:00.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:00 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539558, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.62@KRBTEST.COM for kudu/127.29.228.61@KRBTEST.COM
20:06:00.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:00.939610 30704 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } has no permanent_uuid. Determining permanent_uuid...
20:06:00.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.947892 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:01.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:01.006358 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 1
20:06:01.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:01.012344 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:01.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:01.093874 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 2
20:06:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:01.099994 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:01.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:01.185616 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 3
20:06:01.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:01.186291 30783 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 5
20:06:01.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:01.192435 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:01.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:01.193305 30783 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:01.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:01.333050 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 4
20:06:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:01.339144 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:01.622660 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 5
20:06:01.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:01.635830 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:01.740830 30783 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 6
20:06:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:01.753733 30783 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:02.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.188372 30704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } attempt: 6
20:06:02.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.891386 30798 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:02.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:00.893630 30800 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:02.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:02.198598 30704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.228.60:38133: Network error: Client connection negotiation failed: client connection to 127.29.228.60:38133: connect: Connection refused (error 111)
20:06:02.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.198768 30789 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:06:02.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:02.198736 30799 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1308 milliseconds
20:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.203271 30789 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:06:02.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.205632 30789 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:06:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.207022 30789 hybrid_clock.cc:648] HybridClock initialized: now 1790539562207013 us; error 64 us; skew 500 ppm
20:06:02.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.207906 30789 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:06:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:02 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539562, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:06:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.214078 30789 init.cc:377] Logged in from keytab as kudu/127.29.228.60@KRBTEST.COM (short username kudu)
20:06:02.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.218623 30789 webserver.cc:460] Webserver started at http://127.29.228.60:45705/ using document root <none> and password file <none>
20:06:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.219445 30789 fs_manager.cc:359] Metadata directory not provided
20:06:02.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.219619 30789 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:06:02.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.220023 30789 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:06:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.224584 30789 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data/instance:
20:06:02.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18cf3a9603d04c56a8e73a142de15c55"
20:06:02.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:06:02 on dist-test-slave-3vlk"
20:06:02.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.225565 30789 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal/instance:
20:06:02.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18cf3a9603d04c56a8e73a142de15c55"
20:06:02.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:06:02 on dist-test-slave-3vlk"
20:06:02.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.232573 30789 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
20:06:02.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.237887 30817 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:06:02.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.238849 30789 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:06:02.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.239200 30789 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal
20:06:02.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18cf3a9603d04c56a8e73a142de15c55"
20:06:02.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:06:02 on dist-test-slave-3vlk"
20:06:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.239548 30789 fs_report.cc:383] FS layout report
20:06:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:06:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal
20:06:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal
20:06:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data/data
20:06:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:06:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:06:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:06:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:06:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:06:02.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:06:02.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:06:02.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:06:02.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:06:02.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:06:02.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:06:02.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.293975 30789 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:06:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.299641 30789 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:06:02.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.300134 30789 kserver.cc:163] Server-wide thread pool size limit: 3276
20:06:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.380393 30789 rpc_server.cc:307] RPC server started. Bound to: 127.29.228.60:38133
20:06:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.380509 30868 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.228.60:38133 every 8 connection(s)
20:06:02.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.383072 30789 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data/info.pb
20:06:02.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.389520 30869 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:06:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.391479 30608 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 30789
20:06:02.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.391916 30608 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal/instance
20:06:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.406117 30869 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } has no permanent_uuid. Determining permanent_uuid...
20:06:02.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:02 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539562, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.60@KRBTEST.COM for kudu/127.29.228.62@KRBTEST.COM
20:06:02.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:02 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.228.60@KRBTEST.COM
20:06:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.448817 30869 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } has no permanent_uuid. Determining permanent_uuid...
20:06:02.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:02 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539562, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.60@KRBTEST.COM for kudu/127.29.228.61@KRBTEST.COM
20:06:02.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:02 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.228.61@KRBTEST.COM
20:06:02.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.475399 30869 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } has no permanent_uuid. Determining permanent_uuid...
20:06:02.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:02 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539562, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.60@KRBTEST.COM for kudu/127.29.228.60@KRBTEST.COM
20:06:02.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:02 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.228.62@KRBTEST.COM
20:06:02.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.505661 30869 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: Bootstrap starting.
20:06:02.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.508966 30608 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
20:06:02.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.511441 30869 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: Neither blocks nor log segments found. Creating new log.
20:06:02.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.513403 30869 log.cc:824] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: Log is configured to *not* fsync() on all Append() calls
20:06:02.516 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:06:02.517 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:06:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.518342 30869 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: No bootstrap required, opened a new log
20:06:02.527 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:06:02.528 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:06:02.531 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.29.228.62"
port: 44523
}
bound_http_address {
host: "127.29.228.62"
port: 44779
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.29.228.61"
port: 33285
}
bound_http_address {
host: "127.29.228.61"
port: 38609
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.29.228.60"
port: 38133
}
bound_http_address {
host: "127.29.228.60"
port: 45705
}
}
}
20:06:02.532 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:06:02.535 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:06:02.535 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:06:02.536 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:06:02.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.549834 30869 raft_consensus.cc:348] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:02.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.550662 30869 raft_consensus.cc:374] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:06:02.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.550954 30869 raft_consensus.cc:729] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 18cf3a9603d04c56a8e73a142de15c55, State: Initialized, Role: FOLLOWER
20:06:02.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.551818 30869 consensus_queue.cc:260] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:02.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.555995 30879 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } } }
20:06:02.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.556910 30879 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [sys.catalog]: This master's current role is: FOLLOWER
20:06:02.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.558498 30869 sys_catalog.cc:564] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [sys.catalog]: configured and running, proceeding with master startup.
20:06:02.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:02 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539562, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:06:02.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:06:02.576 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:06:02.578 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5cc
20:06:02.585 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:06:02.586 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5cc
20:06:02.588 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:06:02.594 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:06:02.595 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:06:02.596 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:06:02.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:02.621656 30891 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:02.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:02.622023 30891 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:02.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.595660 30789 master_runner.cc:186] Error getting master registration for 127.29.228.62:44523: OK, instance_id { permanent_uuid: "7e67406bc78847358dbeda373a66940e" instance_seqno: 1790539558524288 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:06:02.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.628569 30789 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:06:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:02 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539562, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:06:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:06:02.640 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:06:02.640 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5cc
20:06:02.642 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:06:02.642 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.228.62:44523
20:06:02.644 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:06:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.644554 30608 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu with pid 30634
20:06:02.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:06:02.667 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.228.60:38133
20:06:02.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:06:02.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.668251 30608 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu with pid 30789
20:06:02.689 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:06:02.689 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.228.61:33285
20:06:02.689 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:06:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.689952 30608 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu with pid 30710
20:06:02.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:06:02.711 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.228.62:44523
20:06:02.712 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:06:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:02.714352 30608 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:06:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:06:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/logs
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data/info.pb
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.62:44523
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.62
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44779
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.62.keytab
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.62
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:06:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:06:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:06:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:06:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:06:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:06:02.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:06:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.021147 30895 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:06:03.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.021929 30895 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:06:03.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.022653 30895 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:06:03.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.054345 30895 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:06:03.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.054615 30895 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:06:03.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.055076 30895 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:06:03.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.055438 30895 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:06:03.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.055770 30895 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:06:03.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.056111 30895 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:06:03.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.056370 30895 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38181/jwks.json
20:06:03.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:03.094161 30895 master_runner.cc:386] Master server non-default flags:
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.62.keytab
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.62
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.62:44523
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data/info.pb
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.62
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44779
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/logs
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:06:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:06:03.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:06:03.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:06:03.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:06:03.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:06:03.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:03.095490 30895 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:06:03.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:03.096637 30895 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:06:03.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.112907 30904 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.113857 30902 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:03.114352 30895 server_base.cc:1034] running on GCE node
20:06:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:03.114140 30901 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:04.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.336789 30895 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:06:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.339953 30895 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:06:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.341372 30895 hybrid_clock.cc:648] HybridClock initialized: now 1790539564341344 us; error 47 us; skew 500 ppm
20:06:04.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.342211 30895 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:06:04.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:04 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539564, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:06:04.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.347934 30895 init.cc:377] Logged in from keytab as kudu/127.29.228.62@KRBTEST.COM (short username kudu)
20:06:04.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.352910 30895 webserver.cc:460] Webserver started at http://127.29.228.62:44779/ using document root <none> and password file <none>
20:06:04.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.353834 30895 fs_manager.cc:359] Metadata directory not provided
20:06:04.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.354102 30895 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:06:04.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.362794 30895 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
20:06:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.367937 30914 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:06:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.369093 30895 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:06:04.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.369364 30895 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal
20:06:04.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e67406bc78847358dbeda373a66940e"
20:06:04.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:05:58 on dist-test-slave-3vlk"
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.371836 30895 fs_report.cc:383] FS layout report
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data/data
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:06:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:06:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.404435 30895 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:06:04.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.409582 30895 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:06:04.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.410038 30895 kserver.cc:163] Server-wide thread pool size limit: 3276
20:06:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.479948 30895 rpc_server.cc:307] RPC server started. Bound to: 127.29.228.62:44523
20:06:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.480069 30965 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.228.62:44523 every 8 connection(s)
20:06:04.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.482932 30895 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/data/info.pb
20:06:04.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.484805 30608 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 30895
20:06:04.485 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:06:04.485 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.228.60:38133
20:06:04.486 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:06:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.487401 30608 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:06:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:06:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal
20:06:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data
20:06:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:06:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:06:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/logs
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data/info.pb
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.60:38133
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.60
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45705
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.60.keytab
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.60
20:06:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:06:04.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:06:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:06:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:06:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:06:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:06:04.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:06:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.495453 30966 sys_catalog.cc:263] Verifying existing consensus state
20:06:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 20:06:04.496284 30966 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:06:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.496698 30895 master.cc:537] Master@127.29.228.62:44523 shutting down...
20:06:04.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.551245 30895 master.cc:552] Master@127.29.228.62:44523 shutdown complete.
20:06:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:06:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.798684 30968 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:06:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.799221 30968 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:06:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.799835 30968 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:06:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.831912 30968 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:06:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.832129 30968 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:06:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.832377 30968 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:06:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.832566 30968 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:06:04.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.832922 30968 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:06:04.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.833272 30968 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:06:04.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.833570 30968 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38181/jwks.json
20:06:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.869426 30968 master_runner.cc:386] Master server non-default flags:
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.60.keytab
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.60
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.60:38133
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:06:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:06:04.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:06:04.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:04.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:04.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data/info.pb
20:06:04.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.60
20:06:04.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45705
20:06:04.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/logs
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.870898 30968 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:06:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.872215 30968 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:06:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.890324 30974 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:06.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.891808 30977 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:06.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:04.892426 30975 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:06.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:04.893981 30968 server_base.cc:1034] running on GCE node
20:06:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.170171 30968 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:06:06.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.172603 30968 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:06:06.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.173893 30968 hybrid_clock.cc:648] HybridClock initialized: now 1790539566173864 us; error 47 us; skew 500 ppm
20:06:06.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.174643 30968 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:06:06.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:06 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539566, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:06:06.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.188386 30968 init.cc:377] Logged in from keytab as kudu/127.29.228.60@KRBTEST.COM (short username kudu)
20:06:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.193456 30968 webserver.cc:460] Webserver started at http://127.29.228.60:45705/ using document root <none> and password file <none>
20:06:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.194600 30968 fs_manager.cc:359] Metadata directory not provided
20:06:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.194795 30968 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:06:06.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.203251 30968 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
20:06:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.208243 30987 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:06:06.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.209699 30968 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.210129 30968 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18cf3a9603d04c56a8e73a142de15c55"
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:06:02 on dist-test-slave-3vlk"
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.213001 30968 fs_report.cc:383] FS layout report
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/wal
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data/data
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:06:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:06:06.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.256240 30968 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:06:06.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.262660 30968 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:06:06.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.263232 30968 kserver.cc:163] Server-wide thread pool size limit: 3276
20:06:06.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.340173 30968 rpc_server.cc:307] RPC server started. Bound to: 127.29.228.60:38133
20:06:06.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.340243 31038 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.228.60:38133 every 8 connection(s)
20:06:06.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.343726 30968 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-2/data/info.pb
20:06:06.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.349272 30608 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 30968
20:06:06.351 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:06:06.351 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.228.61:33285
20:06:06.351 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:06:06.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.353430 30608 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:06:06.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/logs
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data/info.pb
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.61:33285
20:06:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.61
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38609
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.61.keytab
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.61
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:06:06.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:06:06.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.371122 31039 sys_catalog.cc:263] Verifying existing consensus state
20:06:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.402882 31039 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: Bootstrap starting.
20:06:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.434751 31039 log.cc:824] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: Log is configured to *not* fsync() on all Append() calls
20:06:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.439863 31039 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:06:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.440446 31039 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: Bootstrap complete.
20:06:06.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.464937 31039 raft_consensus.cc:348] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.467180 31039 raft_consensus.cc:374] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:06:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.467371 31039 raft_consensus.cc:729] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 18cf3a9603d04c56a8e73a142de15c55, State: Initialized, Role: FOLLOWER
20:06:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.468139 31039 consensus_queue.cc:260] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.471184 31043 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } } }
20:06:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.472000 31043 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [sys.catalog]: This master's current role is: FOLLOWER
20:06:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.472970 31039 sys_catalog.cc:564] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [sys.catalog]: configured and running, proceeding with master startup.
20:06:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.503472 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.504071 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:06.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.519501 30993 tablet.cc:2369] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:06:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.498234 30968 master_runner.cc:186] Error getting master registration for 127.29.228.62:44523: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111),
20:06:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.530268 30968 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:06:06.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.712695 31041 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:06:06.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.713243 31041 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:06:06.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.714002 31041 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:06:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.753419 31041 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:06:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.753654 31041 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:06:06.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.754094 31041 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:06:06.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.754478 31041 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:06:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.754868 31041 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:06:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.755225 31041 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:06:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.755456 31041 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38181/jwks.json
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.794792 31041 master_runner.cc:386] Master server non-default flags:
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.228.20:34737
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.228.62:44523,127.29.228.61:33285,127.29.228.60:38133
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/krb5kdc/kudu_127.29.228.61.keytab
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.228.61
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.228.61:33285
20:06:06.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:06.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:06:06.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38181/jwks.json
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data/info.pb
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.228.61
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38609
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/logs
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 18:48:27 UTC on ccycloud.quasar-hznzdm.root.comops.site
20:06:06.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676202
20:06:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:06:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.796356 31041 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:06:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.797704 31041 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:06:06.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.845965 31043 raft_consensus.cc:482] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:06:06.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.846565 31043 raft_consensus.cc:504] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:06.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.848739 31063 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:06.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.853312 30989 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111)
20:06:06.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.858698 31043 leader_election.cc:288] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523), 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285)
20:06:06.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.870229 30989 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523): Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111)
20:06:06.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:06.873100 30988 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285): Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:06:06.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.873535 30988 leader_election.cc:302] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18cf3a9603d04c56a8e73a142de15c55; no voters: 2680642a507e428b8b566bbc8bb47c24, 7e67406bc78847358dbeda373a66940e
20:06:06.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:06.874480 31043 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:06:07.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:07.372736 31035 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 30969
20:06:07.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:07.373446 31035 debug-util.cc:398] Leaking SignalData structure 0x7b08000737e0 after lost signal to thread 31038
20:06:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:07.508786 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:07.509114 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:07.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:07.854044 30111 debug-util.cc:398] Leaking SignalData structure 0x7b08000a3280 after lost signal to thread 30043
20:06:07.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:07.855098 30111 debug-util.cc:398] Leaking SignalData structure 0x7b08000a3240 after lost signal to thread 30114
20:06:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:08.513239 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:08.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:08.516044 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:08.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:08.233249 31061 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 31041
20:06:08.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:08.509816 31062 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:08.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:08.507285 31041 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.661s user 0.423s sys 0.781s
20:06:08.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:08.509397 31065 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:08.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:08.595028 31041 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.749s user 0.423s sys 0.781s
20:06:08.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.595335 31041 server_base.cc:1034] running on GCE node
20:06:08.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.596764 31041 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:06:08.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.609288 31041 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:06:08.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.625100 31041 hybrid_clock.cc:648] HybridClock initialized: now 1790539568625009 us; error 102 us; skew 500 ppm
20:06:08.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.626739 31041 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:06:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:08 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539568, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:06:08.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.634469 31041 init.cc:377] Logged in from keytab as kudu/127.29.228.61@KRBTEST.COM (short username kudu)
20:06:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.664100 31041 webserver.cc:460] Webserver started at http://127.29.228.61:38609/ using document root <none> and password file <none>
20:06:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.665516 31041 fs_manager.cc:359] Metadata directory not provided
20:06:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.665998 31041 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:06:08.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.682930 31041 fs_manager.cc:709] Time spent opening directory manager: real 0.012s user 0.010s sys 0.002s
20:06:08.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.702687 31075 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:06:08.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.714543 31041 fs_manager.cc:725] Time spent opening block manager: real 0.029s user 0.007s sys 0.011s
20:06:08.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.715034 31041 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data,/tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal
20:06:08.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2680642a507e428b8b566bbc8bb47c24"
20:06:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:06:00 on dist-test-slave-3vlk"
20:06:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.718804 31041 fs_report.cc:383] FS layout report
20:06:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:06:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal
20:06:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal
20:06:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data/data
20:06:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:06:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:06:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:06:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:06:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:06:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:06:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:06:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:06:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:06:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:06:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:06:08.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.747103 31076 raft_consensus.cc:482] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:06:08.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.747624 31076 raft_consensus.cc:504] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.764827 31076 leader_election.cc:288] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523), 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285)
20:06:08.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:08.773346 30988 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285): Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:06:08.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:08.786301 30989 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523): Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111)
20:06:08.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.787024 30989 leader_election.cc:302] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18cf3a9603d04c56a8e73a142de15c55; no voters: 2680642a507e428b8b566bbc8bb47c24, 7e67406bc78847358dbeda373a66940e
20:06:08.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.787945 31076 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:06:08.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.831188 31041 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:06:08.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.849345 31041 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:06:08.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:08.849964 31041 kserver.cc:163] Server-wide thread pool size limit: 3276
20:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:09.228319 31130 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.228.61:33285 every 8 connection(s)
20:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:09.228865 31041 rpc_server.cc:307] RPC server started. Bound to: 127.29.228.61:33285
20:06:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:09.232841 31041 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/data/info.pb
20:06:09.235 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:06:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:09.234632 30608 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu as pid 31041
20:06:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:09.257313 31131 sys_catalog.cc:263] Verifying existing consensus state
20:06:09.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 20:06:09.258299 31131 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:06:09.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:09.258721 31041 master.cc:537] Master@127.29.228.61:33285 shutting down...
20:06:09.351 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:09.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:09.386587 31041 master.cc:552] Master@127.29.228.61:33285 shutdown complete.
20:06:09.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskEMvaJ5/test-tmp/mini-kudu-cluster7657210437416701375/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:06:09.468 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.228.60:38133(127.29.228.60:38133)] connecting to peer
20:06:09.471 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:09.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:09.520347 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:09.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:09.520854 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:09.539 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.29.228.60:38133(127.29.228.60:38133)] Successfully connected to peer
20:06:09.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:09.564 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:09.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:09.567 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:09.568 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:09.568 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:09.666 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:06:09.677 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.29.228.60
20:06:09.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:09 dist-test-slave-3vlk krb5kdc[30620](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539562, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.228.60@KRBTEST.COM
20:06:09.986 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x0bef8bd1, L:/127.0.0.1:53790 - R:/127.29.228.60:38133] using SASL/GSSAPI
20:06:10.009 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:10.031 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:10.036 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:10.038 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.038 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.041 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.041 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.047 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:10.051 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:10.052 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:10.054 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.054 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.055 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.055 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.055 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.057 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.061 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:10.071 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:10.071 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:10.073 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.073 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.080 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.081 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:10.090 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:10.091 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:10.092 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.096 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.096 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.097 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:10.131 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:10.133 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:10.135 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.135 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.135 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.139 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.140 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.140 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:10.191 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:10.192 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:10.194 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.195 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.195 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.197 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.197 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.200 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:10.231 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:10.233 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:10.236 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.236 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.236 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.239 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.240 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:10.391 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:10.393 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:10.398 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.398 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.399 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.400 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.400 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.401 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:10.524140 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:10.524402 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:10.851 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:10.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:10.854 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.855 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.855 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.855 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.855 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:10.855 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:10.857 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:11.153697 31137 raft_consensus.cc:482] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:06:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:11.154152 31137 raft_consensus.cc:504] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:11.155812 31137 leader_election.cc:288] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523), 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285)
20:06:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:11.160593 30989 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523): Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111)
20:06:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:11.160974 30988 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285): Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:06:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:11.161267 30988 leader_election.cc:302] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18cf3a9603d04c56a8e73a142de15c55; no voters: 2680642a507e428b8b566bbc8bb47c24, 7e67406bc78847358dbeda373a66940e
20:06:11.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:11.161757 31137 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:06:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:11.527252 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:11.527487 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:11.532265 30968 master_runner.cc:186] Error getting master registration for 127.29.228.62:44523: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111),
20:06:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:11.532572 30968 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:06:11.591 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:11.592 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:11.595 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:11.595 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:11.595 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:11.597 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:11.597 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:11.597 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:11.598 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:12.151 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:12.153 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:12.154 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:12.154 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:12.155 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:12.155 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:12.155 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:12.155 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:12.156 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:12.530354 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:12.530601 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:13.533325 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:13.533553 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:14.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:14.147200 30032 debug-util.cc:398] Leaking SignalData structure 0x7b08000ab700 after lost signal to thread 29967
20:06:14.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:14.147992 30032 debug-util.cc:398] Leaking SignalData structure 0x7b08000ab560 after lost signal to thread 30035
20:06:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:14.537843 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:14.538178 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:15.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:15.046754 31147 raft_consensus.cc:482] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:06:15.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:15.047472 31147 raft_consensus.cc:504] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:15.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:15.049553 31147 leader_election.cc:288] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523), 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285)
20:06:15.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:15.052637 30989 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111) [suppressed 5 similar messages]
20:06:15.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:15.055941 30988 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285): Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:06:15.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:15.056519 30989 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523): Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111)
20:06:15.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:15.056945 30989 leader_election.cc:302] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18cf3a9603d04c56a8e73a142de15c55; no voters: 2680642a507e428b8b566bbc8bb47c24, 7e67406bc78847358dbeda373a66940e
20:06:15.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:15.057729 31147 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:06:15.390 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:15.392 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:15.394 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:15.394 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:15.395 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:15.395 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:15.396 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:15.396 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:15.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:15.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:15.543044 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:15.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:15.543496 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:16.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:16 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539576, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:06:16.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:16.191836 30983 init.cc:295] Successfully reacquired a new kerberos TGT
20:06:16.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:16.535152 30968 master_runner.cc:186] Error getting master registration for 127.29.228.62:44523: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111),
20:06:16.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:16.535538 30968 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:06:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:16.546334 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:16.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:16.546551 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:17.311 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:17.312 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:17.314 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:17.314 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:17.314 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:17.314 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:17.314 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:17.314 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:17.316 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:17.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:17.549144 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:17.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:17.549381 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:18.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:18.377159 31158 raft_consensus.cc:482] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:06:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:18.377499 31158 raft_consensus.cc:504] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:18.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:18.379079 31158 leader_election.cc:288] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523), 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285)
20:06:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:18.383713 30989 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523): Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111)
20:06:18.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:18.384135 30988 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285): Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:06:18.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:18.384531 30988 leader_election.cc:302] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18cf3a9603d04c56a8e73a142de15c55; no voters: 2680642a507e428b8b566bbc8bb47c24, 7e67406bc78847358dbeda373a66940e
20:06:18.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:18.385299 31158 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:06:18.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:18.552078 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:18.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:18.552309 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:19.555222 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:19.555537 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:20.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:20.558993 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:20.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:20.559239 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:20.771 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:20.773 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:20.774 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:20.774 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:20.774 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:20.774 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:20.775 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:20.775 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:20.776 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:21.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:21.537983 30968 master_runner.cc:186] Error getting master registration for 127.29.228.62:44523: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111),
20:06:21.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:21.538293 30968 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:06:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:21.562023 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:21.562253 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:22.565205 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:22.565443 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:23.568019 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:23.568296 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:23.711 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:23.713 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:23.714 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:23.714 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:23.715 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:23.715 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:23.715 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:23.715 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:23.716 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:24.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:24.112931 31035 debug-util.cc:398] Leaking SignalData structure 0x7b08000acdc0 after lost signal to thread 30969
20:06:24.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:24.113451 31035 debug-util.cc:398] Leaking SignalData structure 0x7b08000ada80 after lost signal to thread 31038
20:06:24.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:24.571246 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:24.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:24.571544 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:24.765722 30593 debug-util.cc:398] Leaking SignalData structure 0x7b08000b58a0 after lost signal to thread 30468
20:06:24.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:24.766855 30593 debug-util.cc:398] Leaking SignalData structure 0x7b08000af840 after lost signal to thread 30596
20:06:25.411 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:25.415 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:25.416 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:25.417 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:25.417 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:25.417 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:25.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:25.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:25.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:25.575949 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:25.576355 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:26.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:26 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539586, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:06:26.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:26.194391 30983 init.cc:295] Successfully reacquired a new kerberos TGT
20:06:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:26.569901 30968 master_runner.cc:186] Error getting master registration for 127.29.228.62:44523: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111),
20:06:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:26.570438 30968 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:06:26.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:26.580278 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:26.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:26.580590 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:26.584717 31172 raft_consensus.cc:482] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:06:26.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:26.585235 31172 raft_consensus.cc:504] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:26.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:26.591576 30989 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:06:26.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:26.593525 31172 leader_election.cc:288] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523), 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285)
20:06:26.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:26.601764 30988 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285): Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:06:26.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:26.602659 30989 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523): Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111)
20:06:26.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:26.603166 30989 leader_election.cc:302] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18cf3a9603d04c56a8e73a142de15c55; no voters: 2680642a507e428b8b566bbc8bb47c24, 7e67406bc78847358dbeda373a66940e
20:06:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:26.604288 31172 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:06:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:27.583387 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:27.583614 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:28.586557 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:28.586835 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:28.990 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:28.992 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:28.994 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:28.994 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:28.994 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:28.994 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:28.994 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:28.995 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:28.996 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:29.391 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:29.392 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:29.395 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:29.395 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:29.396 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:29.396 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:29.396 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:29.396 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:29.396 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:29.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:29.590667 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:29.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:29.590958 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:29.710 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:29.712 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:29.713 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:29.713 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:29.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:29.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:29.714 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:29.714 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:29.715 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:30.594806 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:30.595124 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:31.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:31.573127 30968 master_runner.cc:186] Error getting master registration for 127.29.228.62:44523: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111),
20:06:31.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:31.573580 30968 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:06:31.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:31.598990 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:31.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:31.599290 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:32.603093 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:32.603400 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:33.011 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:33.012 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:33.013 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:33.013 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:33.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:33.013 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:33.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:33.013 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:33.015 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:33.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:33.607285 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:33.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:33.607581 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:34.251 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:34.252 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:34.253 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:34.253 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:34.253 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:34.253 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:34.253 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:34.254 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:34.255 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:34.611248 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:34.611572 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:34.804034 31183 raft_consensus.cc:482] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:06:34.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:34.804492 31183 raft_consensus.cc:504] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e67406bc78847358dbeda373a66940e" member_type: VOTER last_known_addr { host: "127.29.228.62" port: 44523 } } peers { permanent_uuid: "2680642a507e428b8b566bbc8bb47c24" member_type: VOTER last_known_addr { host: "127.29.228.61" port: 33285 } } peers { permanent_uuid: "18cf3a9603d04c56a8e73a142de15c55" member_type: VOTER last_known_addr { host: "127.29.228.60" port: 38133 } }
20:06:34.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:34.806241 31183 leader_election.cc:288] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523), 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285)
20:06:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:34.808094 30989 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:06:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:34.811689 30989 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7e67406bc78847358dbeda373a66940e (127.29.228.62:44523): Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111)
20:06:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:34.812136 30988 leader_election.cc:334] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2680642a507e428b8b566bbc8bb47c24 (127.29.228.61:33285): Network error: Client connection negotiation failed: client connection to 127.29.228.61:33285: connect: Connection refused (error 111)
20:06:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:34.812471 30988 leader_election.cc:302] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18cf3a9603d04c56a8e73a142de15c55; no voters: 2680642a507e428b8b566bbc8bb47c24, 7e67406bc78847358dbeda373a66940e
20:06:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:34.813074 31183 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:06:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:35.615068 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:35.615465 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:36.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:06:36 dist-test-slave-3vlk krb5kdc[30620](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539596, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.228.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:06:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:36.197305 30983 init.cc:295] Successfully reacquired a new kerberos TGT
20:06:36.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:36.577158 30968 master_runner.cc:186] Error getting master registration for 127.29.228.62:44523: Network error: Client connection negotiation failed: client connection to 127.29.228.62:44523: connect: Connection refused (error 111),
20:06:36.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:36.577667 30968 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:06:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:36.618062 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:36.618279 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:37.620469 31054 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18cf3a9603d04c56a8e73a142de15c55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:06:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:06:37.620684 31054 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:06:37.891 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.62:44523(127.29.228.62:44523)] connecting to peer
20:06:37.892 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.228.61:33285(127.29.228.61:33285)] connecting to peer
20:06:37.893 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:37.894 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:37.894 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.62:44523(127.29.228.62:44523)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:37.894 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.228.61:33285(127.29.228.61:33285)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:37.894 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.62:44523: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523
20:06:37.894 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.228.61:33285: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
20:06:37.896 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285; will retry
20:06:37.902 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28657), Traces: [0ms] refreshing cache from master, [64ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [310ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [313ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [748ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [755ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [783ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [786ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [791ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [792ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [799ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [801ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [804ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [818ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [824ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [825ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [826ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [838ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [842ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [843ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [880ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [883ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [885ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [886ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [940ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [944ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [945ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [983ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [985ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [986ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1135ms] refreshing cache from master, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [1145ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1146ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1595ms] refreshing cache from master, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [1598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1602ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [1603ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2336ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [2340ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [2343ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2895ms] refreshing cache from master, [2896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [2897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [2897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [2900ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2900ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [2901ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [2901ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [6135ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [6137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [6139ms] too many traces: truncated at 100 traces, deferred=Deferred@872612294(state=PENDING, result=null, callback=(continuation of Deferred@174692545 after retry RPC after error@672685449) -> (continuation of Deferred@1167650182 after retry RPC after error@1773787586) -> (continuation of Deferred@802220964 after retry RPC after error@1171136450) -> (continuation of Deferred@1203256864 after retry RPC after error@1617467612) -> (continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556), errback=(continuation of Deferred@174692545 after retry RPC after error@672685449) -> (continuation of Deferred@1167650182 after retry RPC after error@1773787586) -> (continuation of Deferred@802220964 after retry RPC after error@1171136450) -> (continuation of Deferred@1203256864 after retry RPC after error@1617467612) -> (continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556)))
20:06:37.907 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.29.228.62:44523(127.29.228.62:44523)] disconnecting while in state TERMINATED
20:06:37.908 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.29.228.61:33285(127.29.228.61:33285)] disconnecting while in state TERMINATED
20:06:37.908 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.29.228.60:38133(127.29.228.60:38133)] disconnecting while in state READY
20:06:37.913 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:06:37.915 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.29.228.60:38133(127.29.228.60:38133)] handling channelInactive
20:06:37.916 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.228.60:38133(127.29.228.60:38133)] cleaning up while in state READY due to: connection closed
20:06:37.917 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:06:37.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:37.920187 30608 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu with pid 30968
20:06:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:06:39Z chronyd exiting
20:06:40.125 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:06:40.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:40.126416 29881 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu with pid 30160
20:06:40.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:40.151284 29881 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu with pid 30325
20:06:40.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:40.177106 29881 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu with pid 30467
20:06:40.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:40.202721 29881 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu with pid 29893
20:06:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:40.232579 29881 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu with pid 29963
20:06:40.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:06:40.266489 29881 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskEMvaJ5/build/tsan/bin/kudu with pid 30038
20:06:41.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:06:41Z chronyd exiting
20:06:42.430 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28657), Traces: [0ms] refreshing cache from master, [64ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [310ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [313ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [748ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [755ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [783ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [786ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [791ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [792ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [799ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [801ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [804ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [818ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [824ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [825ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [826ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [838ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [842ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [843ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [880ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [883ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [885ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [886ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [940ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [944ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [945ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [983ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [985ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [986ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1135ms] refreshing cache from master, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [1145ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1146ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1595ms] refreshing cache from master, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [1598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1602ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [1603ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2336ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [2340ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [2343ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2895ms] refreshing cache from master, [2896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [2897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [2897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [2900ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2900ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [2901ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [2901ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [6135ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [6137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [6139ms] too many traces: truncated at 100 traces, deferred=Deferred@872612294(state=PENDING, result=null, callback=(continuation of Deferred@174692545 after retry RPC after error@672685449) -> (continuation of Deferred@1167650182 after retry RPC after error@1773787586) -> (continuation of Deferred@802220964 after retry RPC after error@1171136450) -> (continuation of Deferred@1203256864 after retry RPC after error@1617467612) -> (continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556), errback=(continuation of Deferred@174692545 after retry RPC after error@672685449) -> (continuation of Deferred@1167650182 after retry RPC after error@1773787586) -> (continuation of Deferred@802220964 after retry RPC after error@1171136450) -> (continuation of Deferred@1203256864 after retry RPC after error@1617467612) -> (continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.228.61:33285
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28657), Traces: [0ms] refreshing cache from master, [64ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [310ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [313ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [748ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [755ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [783ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [786ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [791ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [792ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [799ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [801ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [804ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [818ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [824ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [825ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [826ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [838ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [842ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [843ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [880ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [883ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [885ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [886ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [940ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [944ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [945ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [983ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [985ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [986ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1135ms] refreshing cache from master, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [1145ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1146ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1595ms] refreshing cache from master, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [1598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1602ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [1603ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2336ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [2340ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [2343ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2895ms] refreshing cache from master, [2896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [2897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [2897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [2900ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2900ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [2901ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [2901ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [6135ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [6137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [6139ms] too many traces: truncated at 100 traces, deferred=Deferred@872612294(state=PENDING, result=null, callback=(continuation of Deferred@174692545 after retry RPC after error@672685449) -> (continuation of Deferred@1167650182 after retry RPC after error@1773787586) -> (continuation of Deferred@802220964 after retry RPC after error@1171136450) -> (continuation of Deferred@1203256864 after retry RPC after error@1617467612) -> (continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556), errback=(continuation of Deferred@174692545 after retry RPC after error@672685449) -> (continuation of Deferred@1167650182 after retry RPC after error@1773787586) -> (continuation of Deferred@802220964 after retry RPC after error@1171136450) -> (continuation of Deferred@1203256864 after retry RPC after error@1617467612) -> (continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.228.61:33285
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
20:06:43.159 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:06:43.159 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28657), Traces: [0ms] refreshing cache from master, [64ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [310ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [313ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [748ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [755ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [783ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [786ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [791ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [792ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [799ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [801ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [804ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [818ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [824ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [825ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [826ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [838ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [842ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [843ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [880ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [883ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [885ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [886ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [940ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [944ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [945ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [983ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [985ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [986ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1135ms] refreshing cache from master, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [1145ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1146ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1595ms] refreshing cache from master, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [1598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1602ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [1603ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2336ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [2340ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [2343ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2895ms] refreshing cache from master, [2896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [2897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [2897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [2900ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2900ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [2901ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [2901ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [6135ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [6137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [6139ms] too many traces: truncated at 100 traces, deferred=Deferred@872612294(state=PENDING, result=null, callback=(continuation of Deferred@174692545 after retry RPC after error@672685449) -> (continuation of Deferred@1167650182 after retry RPC after error@1773787586) -> (continuation of Deferred@802220964 after retry RPC after error@1171136450) -> (continuation of Deferred@1203256864 after retry RPC after error@1617467612) -> (continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556), errback=(continuation of Deferred@174692545 after retry RPC after error@672685449) -> (continuation of Deferred@1167650182 after retry RPC after error@1773787586) -> (continuation of Deferred@802220964 after retry RPC after error@1171136450) -> (continuation of Deferred@1203256864 after retry RPC after error@1617467612) -> (continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.228.61:33285
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28657), Traces: [0ms] refreshing cache from master, [64ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [310ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [313ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [748ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [755ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [783ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [786ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [791ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [792ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [799ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [801ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [804ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [818ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [824ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [825ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [826ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [838ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [842ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [843ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [880ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [883ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [885ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [886ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [940ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [944ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [945ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [983ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [985ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [986ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1135ms] refreshing cache from master, [1135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [1144ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [1145ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1146ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1595ms] refreshing cache from master, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [1596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [1598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [1600ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [1602ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [1603ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2336ms] refreshing cache from master, [2336ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [2340ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2342ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [2343ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2895ms] refreshing cache from master, [2896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [2897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [2897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [2900ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.61:33285: Network error: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285, [2900ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.62:44523: Network error: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [2901ms] Sub RPC ConnectToMaster: received response from server master-127.29.228.60:38133: OK, [2901ms] delaying RPC due to: Service unavailable: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523, [6135ms] refreshing cache from master, [6135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.62:44523, [6136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.60:38133, [6137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.228.61:33285, [6139ms] too many traces: truncated at 100 traces, deferred=Deferred@872612294(state=PENDING, result=null, callback=(continuation of Deferred@174692545 after retry RPC after error@672685449) -> (continuation of Deferred@1167650182 after retry RPC after error@1773787586) -> (continuation of Deferred@802220964 after retry RPC after error@1171136450) -> (continuation of Deferred@1203256864 after retry RPC after error@1617467612) -> (continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556), errback=(continuation of Deferred@174692545 after retry RPC after error@672685449) -> (continuation of Deferred@1167650182 after retry RPC after error@1773787586) -> (continuation of Deferred@802220964 after retry RPC after error@1171136450) -> (continuation of Deferred@1203256864 after retry RPC after error@1617467612) -> (continuation of Deferred@1221448382 after retry RPC after error@2110317674) -> (continuation of Deferred@2072485069 after retry RPC after error@1594705177) -> (continuation of Deferred@2037593188 after retry RPC after error@1134716052) -> (continuation of Deferred@75823577 after retry RPC after error@831803869) -> (continuation of Deferred@1369740191 after retry RPC after error@24300539) -> (continuation of Deferred@572235837 after retry RPC after error@831657754) -> (continuation of Deferred@1147093724 after retry RPC after error@1399153855) -> (continuation of Deferred@738208934 after retry RPC after error@227267835) -> (continuation of Deferred@872169012 after retry RPC after error@1262806120) -> (continuation of Deferred@1720485050 after retry RPC after error@905160860) -> (continuation of Deferred@1730952584 after retry RPC after error@195551905) -> (continuation of Deferred@151789115 after retry RPC after error@554950862) -> (continuation of Deferred@659900697 after retry RPC after error@332600760) -> (continuation of Deferred@2051484735 after retry RPC after error@1147971113) -> (continuation of Deferred@1241264700 after retry RPC after error@1107918958) -> (continuation of Deferred@23886532 after retry RPC after error@537970990) -> (continuation of Deferred@139817666 after retry RPC after error@106006870) -> (continuation of Deferred@1686726914 after retry RPC after error@1115063556)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.228.62:44523,127.29.228.60:38133,127.29.228.61:33285) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.62:44523(127.29.228.62:44523): Connection refused: /127.29.228.62:44523,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.228.61:33285(127.29.228.61:33285): Connection refused: /127.29.228.61:33285
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.228.61:33285
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)