05:09:05.363 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
05:09:05.376 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqezt4V/test-tmp
05:09:05.379 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:09:05.383 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqezt4V/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:09:05.606 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
05:09:05.757 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:09:05.758 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:09:05.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:05.762331 374 mini_oidc.cc:152] Starting JWKS server
05:09:05.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:05.766438 374 webserver.cc:460] Webserver started at http://127.0.0.1:34995/ using document root <none> and password file <none>
05:09:05.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:05.766907 374 mini_oidc.cc:189] Starting OIDC Discovery server
05:09:05.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:05.769008 374 webserver.cc:460] Webserver started at http://127.0.0.1:38873/ using document root <none> and password file <none>
05:09:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:09:05Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:09:05.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:09:05Z Disabled control of system clock
05:09:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:05.819877 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/wal
05:09:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/data
05:09:05.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:05.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:05.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:05.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:05.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:05.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:05.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/logs
05:09:05.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/data/info.pb
05:09:05.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:05.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:05.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:05.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:05.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:05.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:05.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:05.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:05.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:09:05.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:05.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:05.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:05.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:44555
05:09:05.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
05:09:05.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:05.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:05.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:05.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:05.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:05.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:05.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:05.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:05.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json with env {}
05:09:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.124125 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:06.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.124753 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:06.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.125272 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.155795 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:09:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.156095 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.156281 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:09:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.156612 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:09:06.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.156911 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:09:06.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.157202 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34995/jwks.json
05:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:06.191918 386 master_runner.cc:386] Master server non-default flags:
05:09:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/data
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/wal
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:44555
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:06.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
05:09:06.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:06.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/data/info.pb
05:09:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
05:09:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/logs
05:09:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:09:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:06.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:06.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:06.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:06.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:06.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:06.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:06.193292 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:06.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:06.195338 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:06.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.215930 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:07.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.215970 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:07.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:06.216166 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:06.216352 386 server_base.cc:1034] running on GCE node
05:09:07.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.316659 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:07.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.319640 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.321097 386 hybrid_clock.cc:648] HybridClock initialized: now 1790658547321045 us; error 72 us; skew 500 ppm
05:09:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.322041 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.329130 386 webserver.cc:460] Webserver started at http://127.0.93.190:45041/ using document root <none> and password file <none>
05:09:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.330200 386 fs_manager.cc:359] Metadata directory not provided
05:09:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.330425 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.331017 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:09:07.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.335955 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/data/instance:
05:09:07.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0797cc2a1c7c448f8a4fd242c588423e"
05:09:07.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:07 on dist-test-slave-5kl8"
05:09:07.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.337319 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/wal/instance:
05:09:07.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0797cc2a1c7c448f8a4fd242c588423e"
05:09:07.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:07 on dist-test-slave-5kl8"
05:09:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.345203 386 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
05:09:07.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.350950 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:07.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.352094 386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
05:09:07.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.352483 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/wal
05:09:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0797cc2a1c7c448f8a4fd242c588423e"
05:09:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:07 on dist-test-slave-5kl8"
05:09:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.352917 386 fs_report.cc:383] FS layout report
05:09:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/wal
05:09:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/wal
05:09:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/data/data
05:09:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:07.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.389691 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:07.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.391379 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:07.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.391893 386 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:07.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.475667 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:44555
05:09:07.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.475749 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:44555 every 8 connection(s)
05:09:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.478576 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/data/info.pb
05:09:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.483790 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 386
05:09:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.484323 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-0/wal/instance
05:09:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.485553 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:09:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.489861 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/wal
05:09:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/data
05:09:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/logs
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/data/info.pb
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:46415
05:09:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
05:09:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:07.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:07.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:07.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json with env {}
05:09:07.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.500595 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } has no permanent_uuid. Determining permanent_uuid...
05:09:07.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.534155 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } has no permanent_uuid. Determining permanent_uuid...
05:09:07.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.536512 404 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:46415: connect: Connection refused (error 111)
05:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.541396 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:46415: Network error: Client connection negotiation failed: client connection to 127.0.93.189:46415: connect: Connection refused (error 111)
05:09:07.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.590950 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } attempt: 1
05:09:07.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.594635 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:46415: Network error: Client connection negotiation failed: client connection to 127.0.93.189:46415: connect: Connection refused (error 111)
05:09:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.663141 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } attempt: 2
05:09:07.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.666904 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:46415: Network error: Client connection negotiation failed: client connection to 127.0.93.189:46415: connect: Connection refused (error 111)
05:09:07.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.758394 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } attempt: 3
05:09:07.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.762507 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:46415: Network error: Client connection negotiation failed: client connection to 127.0.93.189:46415: connect: Connection refused (error 111)
05:09:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.800841 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.801647 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.802426 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.837059 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:09:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.837363 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.837551 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:09:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.837726 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:09:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.837910 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:09:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.838042 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34995/jwks.json
05:09:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.872565 456 master_runner.cc:386] Master server non-default flags:
05:09:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/data
05:09:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/wal
05:09:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:46415
05:09:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
05:09:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/data/info.pb
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/logs
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:07.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:07.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:09:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:07.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:07.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.873768 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:07.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.875465 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.893731 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:07.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.905947 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } attempt: 4
05:09:07.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.910055 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:46415: Network error: Client connection negotiation failed: client connection to 127.0.93.189:46415: connect: Connection refused (error 111)
05:09:08.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:08.209422 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } attempt: 5
05:09:08.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:08.212879 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:46415: Network error: Client connection negotiation failed: client connection to 127.0.93.189:46415: connect: Connection refused (error 111)
05:09:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:08.760421 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } attempt: 6
05:09:08.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:08.767241 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:46415: Network error: Client connection negotiation failed: client connection to 127.0.93.189:46415: connect: Connection refused (error 111)
05:09:08.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.893743 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:08.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:07.896466 456 server_base.cc:1034] running on GCE node
05:09:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:07.893877 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.002411 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:09.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.005257 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:09.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.006645 456 hybrid_clock.cc:648] HybridClock initialized: now 1790658549006614 us; error 54 us; skew 500 ppm
05:09:09.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.007645 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.017460 456 webserver.cc:460] Webserver started at http://127.0.93.189:36027/ using document root <none> and password file <none>
05:09:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.018699 456 fs_manager.cc:359] Metadata directory not provided
05:09:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.018949 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.019512 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:09:09.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.024482 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/data/instance:
05:09:09.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1"
05:09:09.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:09 on dist-test-slave-5kl8"
05:09:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.025849 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/wal/instance:
05:09:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1"
05:09:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:09 on dist-test-slave-5kl8"
05:09:09.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.033777 456 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
05:09:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.039719 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:09.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.040949 456 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
05:09:09.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.041327 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/wal
05:09:09.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1"
05:09:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:09 on dist-test-slave-5kl8"
05:09:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.041743 456 fs_report.cc:383] FS layout report
05:09:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/wal
05:09:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/wal
05:09:09.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/data/data
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.078799 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.080390 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.080874 456 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.158958 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:46415
05:09:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.159034 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:46415 every 8 connection(s)
05:09:09.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.162091 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/data/info.pb
05:09:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.163090 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 456
05:09:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.163558 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-1/wal/instance
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.166561 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/wal
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/data
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/logs
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/data/info.pb
05:09:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:09:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:40307
05:09:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
05:09:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json with env {}
05:09:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.169032 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:09:09.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.182489 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } has no permanent_uuid. Determining permanent_uuid...
05:09:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.205900 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } has no permanent_uuid. Determining permanent_uuid...
05:09:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.223824 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } has no permanent_uuid. Determining permanent_uuid...
05:09:09.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.226670 478 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:09.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.231091 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:09.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.280586 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 1
05:09:09.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.284458 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.352870 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 2
05:09:09.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.356669 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:09.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.448056 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 3
05:09:09.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.451542 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:09.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.476930 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:09.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.477468 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:09.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.478137 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:09.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.510581 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:09:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.510880 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.511096 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:09:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.511274 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:09:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.511516 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:09:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.511754 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34995/jwks.json
05:09:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.547014 531 master_runner.cc:386] Master server non-default flags:
05:09:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/data
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/wal
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:40307
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34995/jwks.json
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/data/info.pb
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/logs
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:09:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.548373 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.550227 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:09.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.571185 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:09.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.594915 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 4
05:09:09.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.599015 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:09.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.827767 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } attempt: 7
05:09:09.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.841374 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } has no permanent_uuid. Determining permanent_uuid...
05:09:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.847101 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:09.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.898414 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 5
05:09:09.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.902645 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:09.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.905474 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 1
05:09:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.909217 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:09.990628 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 2
05:09:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.994469 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:10.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.079854 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 3
05:09:10.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:10.084112 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.224520 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 4
05:09:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:10.228535 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:10.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.450034 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 6
05:09:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:10.457371 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:10.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.511941 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 5
05:09:10.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:10.515668 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:40307: Network error: Client connection negotiation failed: client connection to 127.0.93.188:40307: connect: Connection refused (error 111)
05:09:10.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.571278 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:09.571310 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:10.687060 543 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:09:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.687215 531 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:09:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.691622 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.694653 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:10.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.696112 531 hybrid_clock.cc:648] HybridClock initialized: now 1790658550696060 us; error 64 us; skew 500 ppm
05:09:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.696976 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:10.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.703881 531 webserver.cc:460] Webserver started at http://127.0.93.188:37795/ using document root <none> and password file <none>
05:09:10.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.704877 531 fs_manager.cc:359] Metadata directory not provided
05:09:10.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.705080 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:10.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.705502 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:09:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.712994 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/data/instance:
05:09:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b3c8fd3599540d1bebed89372fcb48b"
05:09:10.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:10 on dist-test-slave-5kl8"
05:09:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.714629 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/wal/instance:
05:09:10.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b3c8fd3599540d1bebed89372fcb48b"
05:09:10.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:10 on dist-test-slave-5kl8"
05:09:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.724632 531 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
05:09:10.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.730182 556 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:10.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.731348 531 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
05:09:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.731684 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/wal
05:09:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b3c8fd3599540d1bebed89372fcb48b"
05:09:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:10 on dist-test-slave-5kl8"
05:09:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.732055 531 fs_report.cc:383] FS layout report
05:09:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/wal
05:09:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/wal
05:09:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/data/data
05:09:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.772500 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:10.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.774106 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:10.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.774616 531 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.849905 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:40307
05:09:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.850006 607 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:40307 every 8 connection(s)
05:09:10.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.852754 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/data/info.pb
05:09:10.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.859366 608 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:09:10.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.863070 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 531
05:09:10.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.863445 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/master-2/wal/instance
05:09:10.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.871867 608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } has no permanent_uuid. Determining permanent_uuid...
05:09:10.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.908551 608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } has no permanent_uuid. Determining permanent_uuid...
05:09:10.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.920445 608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } has no permanent_uuid. Determining permanent_uuid...
05:09:10.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.937983 608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b: Bootstrap starting.
05:09:10.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.944392 608 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b: Neither blocks nor log segments found. Creating new log.
05:09:10.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.946209 608 log.cc:824] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b: Log is configured to *not* fsync() on all Append() calls
05:09:10.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.951702 608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b: No bootstrap required, opened a new log
05:09:10.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.974886 608 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } }
05:09:10.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.975973 608 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:09:10.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.976177 608 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b3c8fd3599540d1bebed89372fcb48b, State: Initialized, Role: FOLLOWER
05:09:10.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.976936 608 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } }
05:09:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.979329 617 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } } }
05:09:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.980003 617 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [sys.catalog]: This master's current role is: FOLLOWER
05:09:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:10.981318 608 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [sys.catalog]: configured and running, proceeding with master startup.
05:09:11.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:11.009457 628 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:11.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:11.009840 628 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:11.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.004480 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:44555: OK, instance_id { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" instance_seqno: 1790658547451796 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:09:11.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.029969 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:09:11.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:11.030032 562 tablet.cc:2369] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:09:11.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.068076 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 6
05:09:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.087069 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e: Bootstrap starting.
05:09:11.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.093982 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e: Neither blocks nor log segments found. Creating new log.
05:09:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.095747 454 log.cc:824] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e: Log is configured to *not* fsync() on all Append() calls
05:09:11.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.100482 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e: No bootstrap required, opened a new log
05:09:11.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.121408 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } }
05:09:11.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.122073 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:09:11.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.122335 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0797cc2a1c7c448f8a4fd242c588423e, State: Initialized, Role: FOLLOWER
05:09:11.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.123198 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } }
05:09:11.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.126336 632 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } } }
05:09:11.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.127182 632 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [sys.catalog]: This master's current role is: FOLLOWER
05:09:11.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.127686 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [sys.catalog]: configured and running, proceeding with master startup.
05:09:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:11.151810 408 tablet.cc:2369] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:09:11.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:11.175776 643 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:11.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:11.176492 643 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.161259 386 master_runner.cc:186] Error getting master registration for 127.0.93.189:46415: OK, instance_id { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" instance_seqno: 1790658549135849 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:09:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.206053 386 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:09:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.456504 632 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:09:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.457011 632 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } }
05:09:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.459360 632 leader_election.cc:288] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5d4f1bdf19aa40758865ed6a71ee6dd1 (127.0.93.189:46415), 8b3c8fd3599540d1bebed89372fcb48b (127.0.93.188:40307)
05:09:11.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.461377 583 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0797cc2a1c7c448f8a4fd242c588423e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8b3c8fd3599540d1bebed89372fcb48b" is_pre_election: true
05:09:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.462273 583 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0797cc2a1c7c448f8a4fd242c588423e in term 0.
05:09:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.464098 403 leader_election.cc:302] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0797cc2a1c7c448f8a4fd242c588423e, 8b3c8fd3599540d1bebed89372fcb48b; no voters:
05:09:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.465644 632 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 0 FOLLOWER]: Leader pre-election won for term 1
05:09:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.465975 632 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
05:09:11.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.466293 632 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 0 FOLLOWER]: Advancing to term 1
05:09:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.473973 632 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } }
05:09:11.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.478060 632 leader_election.cc:288] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [CANDIDATE]: Term 1 election: Requested vote from peers 5d4f1bdf19aa40758865ed6a71ee6dd1 (127.0.93.189:46415), 8b3c8fd3599540d1bebed89372fcb48b (127.0.93.188:40307)
05:09:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.478330 583 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0797cc2a1c7c448f8a4fd242c588423e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8b3c8fd3599540d1bebed89372fcb48b"
05:09:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.478919 583 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [term 0 FOLLOWER]: Advancing to term 1
05:09:11.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.497094 583 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0797cc2a1c7c448f8a4fd242c588423e in term 1.
05:09:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.505245 403 leader_election.cc:302] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0797cc2a1c7c448f8a4fd242c588423e, 8b3c8fd3599540d1bebed89372fcb48b; no voters:
05:09:11.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.509464 632 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 1 FOLLOWER]: Leader election won for term 1
05:09:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.517668 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } attempt: 7
05:09:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.527807 632 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [term 1 LEADER]: Becoming Leader. State: Replica: 0797cc2a1c7c448f8a4fd242c588423e, State: Running, Role: LEADER
05:09:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.533944 632 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } }
05:09:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.572286 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1: Bootstrap starting.
05:09:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.584167 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1: Neither blocks nor log segments found. Creating new log.
05:09:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.587589 529 log.cc:824] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1: Log is configured to *not* fsync() on all Append() calls
05:09:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.590560 645 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0797cc2a1c7c448f8a4fd242c588423e. Latest consensus state: current_term: 1 leader_uuid: "0797cc2a1c7c448f8a4fd242c588423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } } }
05:09:11.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.592707 645 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [sys.catalog]: This master's current role is: LEADER
05:09:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.601301 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1: No bootstrap required, opened a new log
05:09:11.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.604527 649 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
05:09:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.617237 649 catalog_manager.cc:1484] Initializing Kudu cluster ID...
05:09:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.643415 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } }
05:09:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.644651 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:09:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.645156 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5d4f1bdf19aa40758865ed6a71ee6dd1, State: Initialized, Role: FOLLOWER
05:09:11.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.646389 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } }
05:09:11.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.655771 583 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [term 1 FOLLOWER]: Refusing update from remote peer 0797cc2a1c7c448f8a4fd242c588423e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:09:11.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.659591 651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } } }
05:09:11.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.662055 651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [sys.catalog]: This master's current role is: FOLLOWER
05:09:11.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.662365 645 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [LEADER]: Connected to new peer: Peer: permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:09:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.668551 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [sys.catalog]: configured and running, proceeding with master startup.
05:09:11.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.668736 503 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0797cc2a1c7c448f8a4fd242c588423e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1"
05:09:11.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.669489 503 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [term 0 FOLLOWER]: Advancing to term 1
05:09:11.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.677166 503 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0797cc2a1c7c448f8a4fd242c588423e in term 1.
05:09:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.680648 502 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [term 1 FOLLOWER]: Refusing update from remote peer 0797cc2a1c7c448f8a4fd242c588423e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:09:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.681907 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0797cc2a1c7c448f8a4fd242c588423e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" is_pre_election: true
05:09:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.683156 504 raft_consensus.cc:2362] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 0797cc2a1c7c448f8a4fd242c588423e in term 1. Re-sending same reply.
05:09:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.684832 645 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [LEADER]: Connected to new peer: Peer: permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:09:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.707110 652 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0797cc2a1c7c448f8a4fd242c588423e. Latest consensus state: current_term: 1 leader_uuid: "0797cc2a1c7c448f8a4fd242c588423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } } }
05:09:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.707931 652 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [sys.catalog]: This master's current role is: FOLLOWER
05:09:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.716069 650 mvcc.cc:205] Tried to move back new op lower bound from 7334537427559399424 to 7334537427130535936. Current Snapshot: MvccSnapshot[applied={T|T < 7334537427559399424}]
05:09:11.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.720096 654 mvcc.cc:205] Tried to move back new op lower bound from 7334537427559399424 to 7334537427130535936. Current Snapshot: MvccSnapshot[applied={T|T < 7334537427559399424}]
05:09:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.726792 651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0797cc2a1c7c448f8a4fd242c588423e. Latest consensus state: current_term: 1 leader_uuid: "0797cc2a1c7c448f8a4fd242c588423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } } }
05:09:11.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.727918 651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [sys.catalog]: This master's current role is: FOLLOWER
05:09:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.729424 632 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0797cc2a1c7c448f8a4fd242c588423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } } }
05:09:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.729548 652 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0797cc2a1c7c448f8a4fd242c588423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } } }
05:09:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.730160 632 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [sys.catalog]: This master's current role is: LEADER
05:09:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.730248 652 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b [sys.catalog]: This master's current role is: FOLLOWER
05:09:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.734516 645 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0797cc2a1c7c448f8a4fd242c588423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } } }
05:09:11.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.735301 645 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e [sys.catalog]: This master's current role is: LEADER
05:09:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.740904 649 catalog_manager.cc:1347] Generated new cluster ID: 5940d3ab9bd4468a952aa4ba7337257e
05:09:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.741221 649 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
05:09:11.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.746647 651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0797cc2a1c7c448f8a4fd242c588423e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0797cc2a1c7c448f8a4fd242c588423e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 44555 } } peers { permanent_uuid: "5d4f1bdf19aa40758865ed6a71ee6dd1" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 46415 } } peers { permanent_uuid: "8b3c8fd3599540d1bebed89372fcb48b" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 40307 } } }
05:09:11.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.747388 651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1 [sys.catalog]: This master's current role is: FOLLOWER
05:09:11.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.784705 649 catalog_manager.cc:1370] Generated new certificate authority record
05:09:11.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.787600 649 catalog_manager.cc:1504] Loading token signing keys...
05:09:11.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.826426 649 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 0797cc2a1c7c448f8a4fd242c588423e: Generated new TSK 0
05:09:11.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.827787 649 catalog_manager.cc:1514] Initializing in-progress tserver states...
05:09:11.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.856880 670 catalog_manager.cc:1259] Loaded cluster ID: 5940d3ab9bd4468a952aa4ba7337257e
05:09:11.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.860486 670 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1: loading cluster ID for follower catalog manager: success
05:09:11.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.869071 670 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1: acquiring CA information for follower catalog manager: success
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.882877 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/wal
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/data
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/logs
05:09:11.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/data/info.pb
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:09:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:11.883985 670 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5d4f1bdf19aa40758865ed6a71ee6dd1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:09:12.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:12.015966 628 catalog_manager.cc:1259] Loaded cluster ID: 5940d3ab9bd4468a952aa4ba7337257e
05:09:12.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:12.016625 628 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b: loading cluster ID for follower catalog manager: success
05:09:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:12.035887 628 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b: acquiring CA information for follower catalog manager: success
05:09:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:12.045938 628 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8b3c8fd3599540d1bebed89372fcb48b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:09:12.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:12.431954 673 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:12.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:12.433001 673 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:12.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:12.434000 673 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:12.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:12.490801 673 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:12.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:12.492553 673 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
05:09:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:12.554977 673 tablet_server_runner.cc:78] Tablet server non-default flags:
05:09:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/data
05:09:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/wal
05:09:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/data/info.pb
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/logs
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:12.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:12.558399 673 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:12.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:12.561141 673 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:12.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:12.614063 680 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:12.626091 683 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:13.981714 681 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:09:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:13.981779 673 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:09:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:12.625236 679 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:13.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:13.986629 673 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:13.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:13.989832 673 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:13.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:13.991495 673 hybrid_clock.cc:648] HybridClock initialized: now 1790658553991452 us; error 66 us; skew 500 ppm
05:09:13.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:13.992476 673 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:13.999467 673 webserver.cc:460] Webserver started at http://127.0.93.129:41367/ using document root <none> and password file <none>
05:09:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.000537 673 fs_manager.cc:359] Metadata directory not provided
05:09:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.000779 673 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.001322 673 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:09:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.007581 673 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/data/instance:
05:09:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04119fd20aaf4942a7f79674acccdb1b"
05:09:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:13 on dist-test-slave-5kl8"
05:09:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.008957 673 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/wal/instance:
05:09:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04119fd20aaf4942a7f79674acccdb1b"
05:09:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:13 on dist-test-slave-5kl8"
05:09:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.016883 673 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
05:09:14.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.023041 689 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:14.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.024276 673 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
05:09:14.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.024688 673 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/wal
05:09:14.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04119fd20aaf4942a7f79674acccdb1b"
05:09:14.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:13 on dist-test-slave-5kl8"
05:09:14.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.025077 673 fs_report.cc:383] FS layout report
05:09:14.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:14.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/wal
05:09:14.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/wal
05:09:14.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/data/data
05:09:14.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:14.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.073972 673 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:14.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.076582 673 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:14.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.077175 673 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:14.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.080273 673 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:09:14.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.085763 673 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:09:14.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.085997 673 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:09:14.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.086300 673 ts_tablet_manager.cc:610] Registered 0 tablets
05:09:14.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.086483 673 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:09:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.290077 673 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:39569
05:09:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.290231 801 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:39569 every 8 connection(s)
05:09:14.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.293097 673 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/data/info.pb
05:09:14.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.302587 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 673
05:09:14.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.303030 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-0/wal/instance
05:09:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.334249 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/wal
05:09:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/data
05:09:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/logs
05:09:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/data/info.pb
05:09:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:09:14.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.355994 802 heartbeater.cc:347] Connected to a master server at 127.0.93.188:40307
05:09:14.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.357538 802 heartbeater.cc:464] Registering TS with master...
05:09:14.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.357595 804 heartbeater.cc:347] Connected to a master server at 127.0.93.190:44555
05:09:14.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.357889 804 heartbeater.cc:464] Registering TS with master...
05:09:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.359157 802 heartbeater.cc:511] Master 127.0.93.188:40307 requested a full tablet report, sending...
05:09:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.360540 803 heartbeater.cc:347] Connected to a master server at 127.0.93.189:46415
05:09:14.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.360867 803 heartbeater.cc:464] Registering TS with master...
05:09:14.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.361680 803 heartbeater.cc:511] Master 127.0.93.189:46415 requested a full tablet report, sending...
05:09:14.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.362995 804 heartbeater.cc:511] Master 127.0.93.190:44555 requested a full tablet report, sending...
05:09:14.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.363078 573 ts_manager.cc:194] Registered new tserver with Master: 04119fd20aaf4942a7f79674acccdb1b (127.0.93.129:39569)
05:09:14.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.365339 419 ts_manager.cc:194] Registered new tserver with Master: 04119fd20aaf4942a7f79674acccdb1b (127.0.93.129:39569)
05:09:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.366102 493 ts_manager.cc:194] Registered new tserver with Master: 04119fd20aaf4942a7f79674acccdb1b (127.0.93.129:39569)
05:09:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.369897 419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:39189
05:09:14.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:14.727614 812 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:14.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:14.728103 812 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:14.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:14.728658 812 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:14.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:14.759534 812 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:14.760426 812 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
05:09:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.794576 812 tablet_server_runner.cc:78] Tablet server non-default flags:
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/data
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/wal
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/data/info.pb
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
05:09:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/logs
05:09:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:09:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.795796 812 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.797812 812 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:14.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:14.818262 819 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:15.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.376372 804 heartbeater.cc:503] Master 127.0.93.190:44555 was elected leader, sending a full tablet report...
05:09:15.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:14.818243 821 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:15.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:14.818255 818 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:15.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:14.819886 812 server_base.cc:1034] running on GCE node
05:09:15.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.920665 812 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:15.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.923343 812 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.924817 812 hybrid_clock.cc:648] HybridClock initialized: now 1790658555924776 us; error 56 us; skew 500 ppm
05:09:15.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.926033 812 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.932658 812 webserver.cc:460] Webserver started at http://127.0.93.130:46765/ using document root <none> and password file <none>
05:09:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.933590 812 fs_manager.cc:359] Metadata directory not provided
05:09:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.933812 812 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.934195 812 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:09:15.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.939105 812 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/data/instance:
05:09:15.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f927bdc4f064a3ea62481365bf3827d"
05:09:15.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:15 on dist-test-slave-5kl8"
05:09:15.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.940454 812 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/wal/instance:
05:09:15.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f927bdc4f064a3ea62481365bf3827d"
05:09:15.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:15 on dist-test-slave-5kl8"
05:09:15.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.947861 812 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
05:09:15.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.953984 829 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:15.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.955075 812 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
05:09:15.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.955432 812 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/wal
05:09:15.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f927bdc4f064a3ea62481365bf3827d"
05:09:15.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:15 on dist-test-slave-5kl8"
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:15.955803 812 fs_report.cc:383] FS layout report
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/wal
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/wal
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/data/data
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:16.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.008021 812 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:16.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.009538 812 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:16.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.010046 812 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:16.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.013033 812 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:09:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.017452 812 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:09:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.017683 812 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:09:16.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.017974 812 ts_tablet_manager.cc:610] Registered 0 tablets
05:09:16.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.018136 812 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:09:16.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.167343 812 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:41977
05:09:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.167433 942 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:41977 every 8 connection(s)
05:09:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.170428 812 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/data/info.pb
05:09:16.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.179541 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 812
05:09:16.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.180226 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-1/wal/instance
05:09:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.200454 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/wal
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/data
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/logs
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/data/info.pb
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
05:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
05:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
05:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:09:16.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.228796 944 heartbeater.cc:347] Connected to a master server at 127.0.93.189:46415
05:09:16.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.229369 944 heartbeater.cc:464] Registering TS with master...
05:09:16.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.230661 944 heartbeater.cc:511] Master 127.0.93.189:46415 requested a full tablet report, sending...
05:09:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.234205 493 ts_manager.cc:194] Registered new tserver with Master: 1f927bdc4f064a3ea62481365bf3827d (127.0.93.130:41977)
05:09:16.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.237247 943 heartbeater.cc:347] Connected to a master server at 127.0.93.188:40307
05:09:16.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.237602 943 heartbeater.cc:464] Registering TS with master...
05:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.238503 943 heartbeater.cc:511] Master 127.0.93.188:40307 requested a full tablet report, sending...
05:09:16.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.241302 573 ts_manager.cc:194] Registered new tserver with Master: 1f927bdc4f064a3ea62481365bf3827d (127.0.93.130:41977)
05:09:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.247884 945 heartbeater.cc:347] Connected to a master server at 127.0.93.190:44555
05:09:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.248190 945 heartbeater.cc:464] Registering TS with master...
05:09:16.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.248901 945 heartbeater.cc:511] Master 127.0.93.190:44555 requested a full tablet report, sending...
05:09:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.250918 419 ts_manager.cc:194] Registered new tserver with Master: 1f927bdc4f064a3ea62481365bf3827d (127.0.93.130:41977)
05:09:16.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.253041 419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:41589
05:09:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:16.555758 953 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:16.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:16.556234 953 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:16.556770 953 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:16.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:16.588243 953 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:16.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:16.589201 953 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.623971 953 tablet_server_runner.cc:78] Tablet server non-default flags:
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:34045
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/data
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/wal
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/data/info.pb
05:09:16.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:44555,127.0.93.189:46415,127.0.93.188:40307
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/logs
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.625547 953 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:16.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.627246 953 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:16.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:16.648137 960 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.256596 945 heartbeater.cc:503] Master 127.0.93.190:44555 was elected leader, sending a full tablet report...
05:09:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:16.651221 953 server_base.cc:1034] running on GCE node
05:09:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:16.648643 959 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:17.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:16.648241 962 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.739594 953 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:17.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.742255 953 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.743717 953 hybrid_clock.cc:648] HybridClock initialized: now 1790658557743682 us; error 52 us; skew 500 ppm
05:09:17.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.744612 953 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:17.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.751677 953 webserver.cc:460] Webserver started at http://127.0.93.131:37675/ using document root <none> and password file <none>
05:09:17.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.752637 953 fs_manager.cc:359] Metadata directory not provided
05:09:17.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.752835 953 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:17.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.753225 953 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:09:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.757913 953 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/data/instance:
05:09:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f86729fff28c4af7a29b9ef0574bc1e4"
05:09:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:17 on dist-test-slave-5kl8"
05:09:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.758987 953 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/wal/instance:
05:09:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f86729fff28c4af7a29b9ef0574bc1e4"
05:09:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:17 on dist-test-slave-5kl8"
05:09:17.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.765947 953 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
05:09:17.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.771799 970 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.772989 953 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
05:09:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.773347 953 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/wal
05:09:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f86729fff28c4af7a29b9ef0574bc1e4"
05:09:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:17 on dist-test-slave-5kl8"
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.773686 953 fs_report.cc:383] FS layout report
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/wal
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/wal
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/data/data
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:17.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.830672 953 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:17.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.832134 953 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:17.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.832612 953 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:17.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.835058 953 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:09:17.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.839186 953 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:09:17.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.839401 953 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:09:17.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.839602 953 ts_tablet_manager.cc:610] Registered 0 tablets
05:09:17.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.839730 953 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:09:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.982143 953 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:38221
05:09:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.982249 1083 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:38221 every 8 connection(s)
05:09:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.984676 953 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/data/info.pb
05:09:17.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.986982 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 953
05:09:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:17.987453 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster3647577203003550665/ts-2/wal/instance
05:09:18.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.040426 1085 heartbeater.cc:347] Connected to a master server at 127.0.93.189:46415
05:09:18.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.040956 1085 heartbeater.cc:464] Registering TS with master...
05:09:18.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.042167 1085 heartbeater.cc:511] Master 127.0.93.189:46415 requested a full tablet report, sending...
05:09:18.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.042546 1084 heartbeater.cc:347] Connected to a master server at 127.0.93.188:40307
05:09:18.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.042881 1084 heartbeater.cc:464] Registering TS with master...
05:09:18.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.043521 1086 heartbeater.cc:347] Connected to a master server at 127.0.93.190:44555
05:09:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.043689 1084 heartbeater.cc:511] Master 127.0.93.188:40307 requested a full tablet report, sending...
05:09:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.043836 1086 heartbeater.cc:464] Registering TS with master...
05:09:18.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.044703 1086 heartbeater.cc:511] Master 127.0.93.190:44555 requested a full tablet report, sending...
05:09:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.046728 493 ts_manager.cc:194] Registered new tserver with Master: f86729fff28c4af7a29b9ef0574bc1e4 (127.0.93.131:38221)
05:09:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.047605 572 ts_manager.cc:194] Registered new tserver with Master: f86729fff28c4af7a29b9ef0574bc1e4 (127.0.93.131:38221)
05:09:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.047726 418 ts_manager.cc:194] Registered new tserver with Master: f86729fff28c4af7a29b9ef0574bc1e4 (127.0.93.131:38221)
05:09:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.049574 418 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:58957
05:09:18.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.069511 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
05:09:18.080 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:09:18.081 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
05:09:18.093 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 44555
}
bound_http_address {
host: "127.0.93.190"
port: 45041
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 46415
}
bound_http_address {
host: "127.0.93.189"
port: 36027
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 40307
}
bound_http_address {
host: "127.0.93.188"
port: 37795
}
}
}
05:09:18.100 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:09:18.103 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 39569
}
bound_http_address {
host: "127.0.93.129"
port: 41367
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 41977
}
bound_http_address {
host: "127.0.93.130"
port: 46765
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 38221
}
bound_http_address {
host: "127.0.93.131"
port: 37675
}
}
}
05:09:18.104 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
05:09:18.299 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
05:09:18.414 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqezt4V/test-tmp
05:09:18.414 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:09:18.414 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqezt4V/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:09:18.419 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
05:09:18.739 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:18.739 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:09:18.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.742280 1096 mini_oidc.cc:152] Starting JWKS server
05:09:18.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.746256 1096 webserver.cc:460] Webserver started at http://127.0.0.1:42637/ using document root <none> and password file <none>
05:09:18.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.746754 1096 mini_oidc.cc:189] Starting OIDC Discovery server
05:09:18.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:18.748492 1096 webserver.cc:460] Webserver started at http://127.0.0.1:39003/ using document root <none> and password file <none>
05:09:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
05:09:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/principal' for realm 'KRBTEST.COM',
05:09:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
05:09:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:18 dist-test-slave-5kl8 krb5kdc[1108](info): setting up network...
05:09:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
05:09:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:18 dist-test-slave-5kl8 krb5kdc[1108](info): set up 2 sockets
05:09:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:18 dist-test-slave-5kl8 krb5kdc[1108](info): commencing operation
05:09:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
05:09:19.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:19.053531 1086 heartbeater.cc:503] Master 127.0.93.190:44555 was elected leader, sending a full tablet report...
05:09:19.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:19.762578 1096 mini_kdc.cc:121] Time spent starting KDC: real 0.989s user 0.000s sys 0.019s
05:09:19.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
05:09:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:09:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
05:09:19.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
05:09:19.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:09:19.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
05:09:19.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
05:09:19.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:09:19.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
05:09:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:09:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/test-user.keytab.
05:09:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/test-user.keytab.
05:09:19.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:19 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658559, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:19.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:09:19.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:09:19Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:09:19.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:09:19Z Disabled control of system clock
05:09:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.62@KRBTEST.COM; defaulting to no policy
05:09:19.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:09:19.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.62@KRBTEST.COM" created.
05:09:19.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:09:19.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.62.keytab.
05:09:19.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.62.keytab.
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:19.952219 1096 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/logs
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data/info.pb
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.62:43233
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.62
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.62.keytab
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.62
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:09:20.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.264879 1122 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:20.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.265421 1122 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:20.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.266002 1122 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.299417 1122 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:09:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.299638 1122 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:09:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.299925 1122 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.300112 1122 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:09:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.300302 1122 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:09:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.300541 1122 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:09:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.300693 1122 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42637/jwks.json
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:20.336799 1122 master_runner.cc:386] Master server non-default flags:
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.62.keytab
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.62
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.62:43233
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data/info.pb
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.62
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/logs
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:20.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:09:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:20.337916 1122 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:20.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:20.338871 1122 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.358026 1131 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.358033 1129 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:20.360031 1122 server_base.cc:1034] running on GCE node
05:09:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:20.358012 1128 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.494104 1122 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.497247 1122 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.498687 1122 hybrid_clock.cc:648] HybridClock initialized: now 1790658561498646 us; error 62 us; skew 500 ppm
05:09:21.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.499502 1122 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:21.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:21 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658561, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:21.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.509050 1122 init.cc:377] Logged in from keytab as kudu/127.1.18.62@KRBTEST.COM (short username kudu)
05:09:21.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.514477 1122 webserver.cc:460] Webserver started at http://127.1.18.62:34259/ using document root <none> and password file <none>
05:09:21.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.515436 1122 fs_manager.cc:359] Metadata directory not provided
05:09:21.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.515651 1122 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.516142 1122 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:09:21.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.521117 1122 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data/instance:
05:09:21.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2cebbb18a87c4a45ab0f0bedff200069"
05:09:21.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:21 on dist-test-slave-5kl8"
05:09:21.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.522254 1122 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal/instance:
05:09:21.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2cebbb18a87c4a45ab0f0bedff200069"
05:09:21.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:21 on dist-test-slave-5kl8"
05:09:21.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.529434 1122 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
05:09:21.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.535235 1139 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:21.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.536397 1122 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.536705 1122 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2cebbb18a87c4a45ab0f0bedff200069"
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:21 on dist-test-slave-5kl8"
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.537005 1122 fs_report.cc:383] FS layout report
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data/data
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.585078 1122 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.591573 1122 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.592015 1122 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:21.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.668216 1122 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.62:43233
05:09:21.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.668286 1190 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.62:43233 every 8 connection(s)
05:09:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.671011 1122 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data/info.pb
05:09:21.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.673842 1096 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 1122
05:09:21.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.674376 1096 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal/instance
05:09:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.677106 1191 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:09:21.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.61@KRBTEST.COM; defaulting to no policy
05:09:21.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:09:21.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.61@KRBTEST.COM" created.
05:09:21.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.689937 1191 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } has no permanent_uuid. Determining permanent_uuid...
05:09:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:09:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.61.keytab.
05:09:21.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.61.keytab.
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.718114 1096 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/logs
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data/info.pb
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.61:46831
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.61
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.61.keytab
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.61
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:09:21.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:21 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658561, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.62@KRBTEST.COM for kudu/127.1.18.62@KRBTEST.COM
05:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.732929 1191 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } has no permanent_uuid. Determining permanent_uuid...
05:09:21.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:21.735477 1143 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:21.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:21.740060 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.61:46831: Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:21.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.789633 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } attempt: 1
05:09:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:21.793277 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.61:46831: Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.861654 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } attempt: 2
05:09:21.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:21.865728 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.61:46831: Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:21.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:21.957162 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } attempt: 3
05:09:21.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:21.960811 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.61:46831: Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:22.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.033535 1197 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:22.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.034062 1197 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:22.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.034652 1197 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.068320 1197 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:09:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.068576 1197 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:09:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.068879 1197 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.069321 1197 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:09:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.069551 1197 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:09:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.069737 1197 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:09:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.069872 1197 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42637/jwks.json
05:09:22.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:22.104218 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } attempt: 4
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.108037 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.61:46831: Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:22.108168 1197 master_runner.cc:386] Master server non-default flags:
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.61.keytab
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.61
05:09:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.61:46831
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data/info.pb
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.61
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/logs
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:22.109678 1197 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:22.111121 1197 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:22.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:22.407497 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } attempt: 5
05:09:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.411201 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.61:46831: Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:22.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:22.958709 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } attempt: 6
05:09:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.965901 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.61:46831: Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.130541 1204 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:22.133235 1197 server_base.cc:1034] running on GCE node
05:09:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.130579 1205 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:23.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:22.131206 1207 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:23.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.265661 1197 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:23.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.270022 1197 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:23.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.271440 1197 hybrid_clock.cc:648] HybridClock initialized: now 1790658563271406 us; error 39 us; skew 500 ppm
05:09:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.272276 1197 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:23.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:23 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658563, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.278457 1197 init.cc:377] Logged in from keytab as kudu/127.1.18.61@KRBTEST.COM (short username kudu)
05:09:23.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.283885 1197 webserver.cc:460] Webserver started at http://127.1.18.61:43289/ using document root <none> and password file <none>
05:09:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.284894 1197 fs_manager.cc:359] Metadata directory not provided
05:09:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.285084 1197 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.285480 1197 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:09:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.290242 1197 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data/instance:
05:09:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a07e9956d3454ce2aaf019323f695f8f"
05:09:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:23 on dist-test-slave-5kl8"
05:09:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.291486 1197 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal/instance:
05:09:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a07e9956d3454ce2aaf019323f695f8f"
05:09:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:23 on dist-test-slave-5kl8"
05:09:23.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.299516 1197 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
05:09:23.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.305017 1218 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.306069 1197 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
05:09:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.306421 1197 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal
05:09:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a07e9956d3454ce2aaf019323f695f8f"
05:09:23.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:23 on dist-test-slave-5kl8"
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.306761 1197 fs_report.cc:383] FS layout report
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data/data
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:23.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.381893 1197 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:23.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.387794 1197 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:23.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.388199 1197 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:23.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.470851 1197 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.61:46831
05:09:23.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.470922 1269 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.61:46831 every 8 connection(s)
05:09:23.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.473655 1197 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data/info.pb
05:09:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.479354 1096 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 1197
05:09:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.479799 1096 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal/instance
05:09:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.482000 1270 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:09:23.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.495000 1270 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } has no permanent_uuid. Determining permanent_uuid...
05:09:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.60@KRBTEST.COM; defaulting to no policy
05:09:23.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:09:23.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.60@KRBTEST.COM" created.
05:09:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:23 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658563, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.61@KRBTEST.COM for kudu/127.1.18.62@KRBTEST.COM
05:09:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:09:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.60.keytab.
05:09:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.60.keytab.
05:09:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.524382 1096 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal
05:09:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data
05:09:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/logs
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data/info.pb
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.60:46141
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.60
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.60.keytab
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.60
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:09:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.532207 1270 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } has no permanent_uuid. Determining permanent_uuid...
05:09:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:23 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658563, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.61@KRBTEST.COM for kudu/127.1.18.61@KRBTEST.COM
05:09:23.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.561465 1270 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } has no permanent_uuid. Determining permanent_uuid...
05:09:23.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.563679 1222 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.567925 1270 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.617372 1270 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 1
05:09:23.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.620831 1270 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:23.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.689204 1270 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 2
05:09:23.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.692844 1270 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:23.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.784233 1270 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 3
05:09:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.787734 1270 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:23.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.846765 1276 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:23.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.847327 1276 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.847900 1276 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.894044 1276 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:09:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.894330 1276 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:09:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.894632 1276 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.894891 1276 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:09:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.895090 1276 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:09:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.895323 1276 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:09:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.895473 1276 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42637/jwks.json
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.931131 1270 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 4
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.931362 1276 master_runner.cc:386] Master server non-default flags:
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.60.keytab
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.60
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.60:46141
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data/info.pb
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.60
05:09:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/logs
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:09:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.932741 1276 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:23.933977 1276 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.935119 1270 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:23.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.954471 1284 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:24.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:24.026283 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } attempt: 7
05:09:24.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:24 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658561, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.62@KRBTEST.COM for kudu/127.1.18.61@KRBTEST.COM
05:09:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:24.051990 1191 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } has no permanent_uuid. Determining permanent_uuid...
05:09:24.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:24.057741 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:24.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:24.116269 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 1
05:09:24.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:24.121444 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:24.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:24.202805 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 2
05:09:24.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:24.206728 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:24.234501 1270 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 5
05:09:24.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:24.238940 1270 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:24.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:24.292105 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 3
05:09:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:24.296391 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:24.436800 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 4
05:09:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:24.441859 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:24.725275 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 5
05:09:24.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:24.729651 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:24.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:24.786420 1270 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 6
05:09:24.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:24.797170 1270 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:25.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.954504 1285 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:25.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:23.954468 1287 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:25.110013 1286 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:09:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.110140 1276 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:09:25.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.113329 1276 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.115887 1276 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.117287 1276 hybrid_clock.cc:648] HybridClock initialized: now 1790658565117227 us; error 52 us; skew 500 ppm
05:09:25.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.118165 1276 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:25.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:25 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658565, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:25.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.125308 1276 init.cc:377] Logged in from keytab as kudu/127.1.18.60@KRBTEST.COM (short username kudu)
05:09:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.131740 1276 webserver.cc:460] Webserver started at http://127.1.18.60:35719/ using document root <none> and password file <none>
05:09:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.132820 1276 fs_manager.cc:359] Metadata directory not provided
05:09:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.133061 1276 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:25.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.133566 1276 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:09:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.138315 1276 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data/instance:
05:09:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdd24dfef5e941b89b299c1fa6cb1b82"
05:09:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:25 on dist-test-slave-5kl8"
05:09:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.139477 1276 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal/instance:
05:09:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdd24dfef5e941b89b299c1fa6cb1b82"
05:09:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:25 on dist-test-slave-5kl8"
05:09:25.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.147447 1276 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
05:09:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.153642 1300 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.154773 1276 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
05:09:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.155205 1276 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal
05:09:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdd24dfef5e941b89b299c1fa6cb1b82"
05:09:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:25 on dist-test-slave-5kl8"
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.155644 1276 fs_report.cc:383] FS layout report
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data/data
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:25.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.203613 1276 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.209594 1276 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:25.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.210037 1276 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.282045 1191 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } attempt: 6
05:09:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:25.291862 1191 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.60:46141: Network error: Client connection negotiation failed: client connection to 127.1.18.60:46141: connect: Connection refused (error 111)
05:09:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.296832 1276 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.60:46141
05:09:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.297328 1353 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.60:46141 every 8 connection(s)
05:09:25.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.299528 1276 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data/info.pb
05:09:25.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.305361 1354 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:09:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.309818 1096 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 1276
05:09:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.310308 1096 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal/instance
05:09:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.322045 1354 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } has no permanent_uuid. Determining permanent_uuid...
05:09:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:25 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658565, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.60@KRBTEST.COM for kudu/127.1.18.62@KRBTEST.COM
05:09:25.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:25 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658559, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.60@KRBTEST.COM
05:09:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.358495 1354 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } has no permanent_uuid. Determining permanent_uuid...
05:09:25.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:25 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658565, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.60@KRBTEST.COM for kudu/127.1.18.61@KRBTEST.COM
05:09:25.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:25 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658559, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.61@KRBTEST.COM
05:09:25.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.387406 1354 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } has no permanent_uuid. Determining permanent_uuid...
05:09:25.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:25 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658565, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.60@KRBTEST.COM for kudu/127.1.18.60@KRBTEST.COM
05:09:25.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:25 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658559, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.62@KRBTEST.COM
05:09:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.417532 1354 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: Bootstrap starting.
05:09:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.424186 1354 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: Neither blocks nor log segments found. Creating new log.
05:09:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.425688 1096 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
05:09:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.426661 1354 log.cc:824] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: Log is configured to *not* fsync() on all Append() calls
05:09:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.430977 1354 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: No bootstrap required, opened a new log
05:09:25.433 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:25.434 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
05:09:25.445 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
05:09:25.446 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
05:09:25.449 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.18.62"
port: 43233
}
bound_http_address {
host: "127.1.18.62"
port: 34259
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.18.61"
port: 46831
}
bound_http_address {
host: "127.1.18.61"
port: 43289
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.18.60"
port: 46141
}
bound_http_address {
host: "127.1.18.60"
port: 35719
}
}
}
05:09:25.449 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:09:25.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:25.450 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:09:25.451 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:09:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.454820 1354 raft_consensus.cc:348] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } }
05:09:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.455588 1354 raft_consensus.cc:374] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:09:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.455986 1354 raft_consensus.cc:729] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bdd24dfef5e941b89b299c1fa6cb1b82, State: Initialized, Role: FOLLOWER
05:09:25.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.456918 1354 consensus_queue.cc:260] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } }
05:09:25.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.460309 1364 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } } }
05:09:25.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.461433 1364 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [sys.catalog]: This master's current role is: FOLLOWER
05:09:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.461977 1354 sys_catalog.cc:564] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [sys.catalog]: configured and running, proceeding with master startup.
05:09:25.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:25.465813 1306 tablet.cc:2369] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:09:25.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:25 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658565, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:25.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:09:25.491 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:25.494 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5cc
05:09:25.506 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:09:25.508 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5cc
05:09:25.511 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:09:25.514 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
05:09:25.516 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:09:25.517 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:09:25.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:25.527827 1376 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:25.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:25.528265 1376 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:25 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658565, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.523099 1276 master_runner.cc:186] Error getting master registration for 127.1.18.62:43233: OK, instance_id { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" instance_seqno: 1790658561647369 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:09:25.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:09:25.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.549949 1276 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:09:25.555 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:25.555 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5cc
05:09:25.556 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:09:25.557 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.62:43233
05:09:25.559 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
05:09:25.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.559891 1096 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu with pid 1122
05:09:25.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:25.587 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.60:46141
05:09:25.588 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
05:09:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.588514 1096 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu with pid 1276
05:09:25.608 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:25.608 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.61:46831
05:09:25.608 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
05:09:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.608829 1096 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu with pid 1197
05:09:25.627 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:25.627 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.62:43233
05:09:25.629 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
05:09:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:25.630435 1096 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal
05:09:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data
05:09:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/logs
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data/info.pb
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.62:43233
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.62
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34259
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.62.keytab
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.62
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:09:25.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:25.810220 798 debug-util.cc:398] Leaking SignalData structure 0x7b08000afa80 after lost signal to thread 674
05:09:25.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:25.811229 798 debug-util.cc:398] Leaking SignalData structure 0x7b08000af600 after lost signal to thread 801
05:09:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:25.985116 1380 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:25.985929 1380 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:25.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:25.986955 1380 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:26.020637 1380 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:09:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:26.020895 1380 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:09:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:26.021289 1380 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:26.021597 1380 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:09:26.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:26.021885 1380 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:09:26.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:26.022181 1380 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:09:26.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:26.022387 1380 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42637/jwks.json
05:09:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:26.059263 1380 master_runner.cc:386] Master server non-default flags:
05:09:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data
05:09:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal
05:09:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:09:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:26.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.62.keytab
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.62
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.62:43233
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data/info.pb
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.62
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34259
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/logs
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:26.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:09:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:26.060779 1380 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:26.062175 1380 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:26.084877 1387 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:26.085167 1388 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:26.086205 1390 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:27.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.272120 1389 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:09:27.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.272169 1380 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:09:27.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.279422 1380 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:27.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.282415 1380 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.283845 1380 hybrid_clock.cc:648] HybridClock initialized: now 1790658567283772 us; error 67 us; skew 500 ppm
05:09:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.284673 1380 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:27 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658567, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.293964 1380 init.cc:377] Logged in from keytab as kudu/127.1.18.62@KRBTEST.COM (short username kudu)
05:09:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.299950 1380 webserver.cc:460] Webserver started at http://127.1.18.62:34259/ using document root <none> and password file <none>
05:09:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.300930 1380 fs_manager.cc:359] Metadata directory not provided
05:09:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.301146 1380 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.309453 1380 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.008s sys 0.000s
05:09:27.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.314373 1400 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:27.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.315619 1380 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
05:09:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.315943 1380 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal
05:09:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2cebbb18a87c4a45ab0f0bedff200069"
05:09:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:21 on dist-test-slave-5kl8"
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.318455 1380 fs_report.cc:383] FS layout report
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data/data
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:27.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.360503 1380 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:27.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.366765 1380 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:27.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.367264 1380 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:27.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.452718 1380 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.62:43233
05:09:27.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.452788 1451 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.62:43233 every 8 connection(s)
05:09:27.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.455381 1380 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/data/info.pb
05:09:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.462631 1096 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 1380
05:09:27.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:27.463 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.60:46141
05:09:27.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.465526 1096 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/logs
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data/info.pb
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.60:46141
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.60
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35719
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.60.keytab
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.60
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:09:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:09:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.466845 1452 sys_catalog.cc:263] Verifying existing consensus state
05:09:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 05:09:27.467803 1452 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:09:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.468281 1380 master.cc:537] Master@127.1.18.62:43233 shutting down...
05:09:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.533015 1380 master.cc:552] Master@127.1.18.62:43233 shutdown complete.
05:09:27.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:09:27.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.776732 1454 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:27.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.777370 1454 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.778050 1454 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:27.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.811343 1454 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:09:27.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.811607 1454 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:09:27.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.812050 1454 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:27.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.812419 1454 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:09:27.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.812734 1454 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:09:27.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.813052 1454 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:09:27.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.813318 1454 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42637/jwks.json
05:09:27.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.859138 1454 master_runner.cc:386] Master server non-default flags:
05:09:27.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:27.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:27.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.60.keytab
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.60
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.60:46141
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data/info.pb
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.60
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35719
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/logs
05:09:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:09:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.860657 1454 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:27.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.862015 1454 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:27.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.883008 1462 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:29.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.884011 1461 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:29.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:27.885309 1454 server_base.cc:1034] running on GCE node
05:09:29.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:27.886195 1464 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:29.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.031637 1454 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:29.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.034370 1454 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:29.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.035801 1454 hybrid_clock.cc:648] HybridClock initialized: now 1790658569035750 us; error 42 us; skew 500 ppm
05:09:29.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.036770 1454 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:29.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:29 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658569, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:29.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.042805 1454 init.cc:377] Logged in from keytab as kudu/127.1.18.60@KRBTEST.COM (short username kudu)
05:09:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.048032 1454 webserver.cc:460] Webserver started at http://127.1.18.60:35719/ using document root <none> and password file <none>
05:09:29.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.049010 1454 fs_manager.cc:359] Metadata directory not provided
05:09:29.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.049228 1454 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.057504 1454 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.008s sys 0.000s
05:09:29.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.062103 1473 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.063325 1454 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:09:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.063724 1454 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal
05:09:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdd24dfef5e941b89b299c1fa6cb1b82"
05:09:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:25 on dist-test-slave-5kl8"
05:09:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.066340 1454 fs_report.cc:383] FS layout report
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/wal
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data/data
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.119424 1454 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:29.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.126000 1454 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:29.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.126537 1454 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:29.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.212585 1454 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.60:46141
05:09:29.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.212725 1524 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.60:46141 every 8 connection(s)
05:09:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.215544 1454 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-2/data/info.pb
05:09:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.220234 1096 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 1454
05:09:29.221 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:29.222 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.61:46831
05:09:29.222 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.223871 1096 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/logs
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data/info.pb
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.61:46831
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.61
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43289
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.61.keytab
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.61
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:29.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:09:29.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.227751 1525 sys_catalog.cc:263] Verifying existing consensus state
05:09:29.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.234320 1525 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: Bootstrap starting.
05:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.269126 1525 log.cc:824] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: Log is configured to *not* fsync() on all Append() calls
05:09:29.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.274755 1525 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
05:09:29.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.275491 1525 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: Bootstrap complete.
05:09:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.302695 1525 raft_consensus.cc:348] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } }
05:09:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.305249 1525 raft_consensus.cc:374] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:09:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.305554 1525 raft_consensus.cc:729] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bdd24dfef5e941b89b299c1fa6cb1b82, State: Initialized, Role: FOLLOWER
05:09:29.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.306349 1525 consensus_queue.cc:260] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } }
05:09:29.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.308997 1529 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } } }
05:09:29.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.309897 1529 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [sys.catalog]: This master's current role is: FOLLOWER
05:09:29.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.310287 1525 sys_catalog.cc:564] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [sys.catalog]: configured and running, proceeding with master startup.
05:09:29.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.338291 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.338963 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.343542 1454 master_runner.cc:186] Error getting master registration for 127.1.18.62:43233: Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111),
05:09:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.373980 1454 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:09:29.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.382129 1479 tablet.cc:2369] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:09:29.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.577740 1527 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:09:29.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.578312 1527 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:09:29.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.578891 1527 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:09:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.612166 1527 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:09:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.612449 1527 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:09:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.612723 1527 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:09:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.612926 1527 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:09:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.613119 1527 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:09:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.613354 1527 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:09:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.613503 1527 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42637/jwks.json
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.651656 1527 master_runner.cc:386] Master server non-default flags:
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.20:34053
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.62:43233,127.1.18.61:46831,127.1.18.60:46141
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/krb5kdc/kudu_127.1.18.61.keytab
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.61
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.61:46831
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42637/jwks.json
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:09:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:09:29.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:09:29.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:09:29.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:09:29.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data/info.pb
05:09:29.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.61
05:09:29.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43289
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/logs
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 04:01:01 UTC on ccycloud.quasar-phqekz.root.comops.site
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679192
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.652905 1527 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:09:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.653896 1527 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:09:29.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.674044 1552 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:29.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.704210 1529 raft_consensus.cc:482] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:09:29.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.704893 1529 raft_consensus.cc:504] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } }
05:09:29.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.710031 1476 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111)
05:09:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.715605 1529 leader_election.cc:288] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233), a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831)
05:09:29.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.729511 1477 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831): Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.734114 1476 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233): Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111)
05:09:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.734598 1476 leader_election.cc:302] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bdd24dfef5e941b89b299c1fa6cb1b82; no voters: 2cebbb18a87c4a45ab0f0bedff200069, a07e9956d3454ce2aaf019323f695f8f
05:09:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:29.735592 1529 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:09:30.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:30.343888 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:30.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:30.344718 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:31.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.674146 1550 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:31.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:29.679296 1549 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:09:31.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:31.235643 1551 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1556 milliseconds
05:09:31.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.235805 1527 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:09:31.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.237458 1527 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:09:31.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.241070 1527 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:09:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.242895 1527 hybrid_clock.cc:648] HybridClock initialized: now 1790658571242836 us; error 154 us; skew 500 ppm
05:09:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.244242 1527 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:09:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:31 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658571, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.252839 1527 init.cc:377] Logged in from keytab as kudu/127.1.18.61@KRBTEST.COM (short username kudu)
05:09:31.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.258759 1527 webserver.cc:460] Webserver started at http://127.1.18.61:43289/ using document root <none> and password file <none>
05:09:31.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.259840 1527 fs_manager.cc:359] Metadata directory not provided
05:09:31.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.260120 1527 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:09:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.270251 1527 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.004s
05:09:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.276191 1562 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:09:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.277843 1527 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.000s
05:09:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.278307 1527 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data,/tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a07e9956d3454ce2aaf019323f695f8f"
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 05:09:23 on dist-test-slave-5kl8"
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.280918 1527 fs_report.cc:383] FS layout report
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data/data
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:09:31.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:09:31.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:09:31.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:09:31.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:09:31.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:09:31.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:09:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.342986 1527 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:09:31.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:31.349681 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:31.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:31.350620 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:31.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.354025 1527 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:09:31.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.354820 1527 kserver.cc:163] Server-wide thread pool size limit: 3276
05:09:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.469010 1527 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.61:46831
05:09:31.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.469900 1613 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.61:46831 every 8 connection(s)
05:09:31.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.473078 1527 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/data/info.pb
05:09:31.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.473894 1096 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu as pid 1527
05:09:31.474 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:09:31.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.523533 1617 raft_consensus.cc:482] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:09:31.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.524056 1617 raft_consensus.cc:504] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } }
05:09:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.526149 1617 leader_election.cc:288] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233), a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831)
05:09:31.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.538475 1614 sys_catalog.cc:263] Verifying existing consensus state
05:09:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 05:09:31.545672 1614 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:09:31.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.550470 1527 master.cc:537] Master@127.1.18.61:46831 shutting down...
05:09:31.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:31.583356 1476 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233): Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111)
05:09:31.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:31 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658569, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.60@KRBTEST.COM for kudu/127.1.18.61@KRBTEST.COM
05:09:31.631 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:31.632555 1477 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831): Remote error: Service unavailable: service kudu.consensus.ConsensusService not registered on Master
05:09:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.633015 1477 leader_election.cc:302] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bdd24dfef5e941b89b299c1fa6cb1b82; no voters: 2cebbb18a87c4a45ab0f0bedff200069, a07e9956d3454ce2aaf019323f695f8f
05:09:31.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.636915 1617 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:09:31.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:31.698120 1527 master.cc:552] Master@127.1.18.61:46831 shutdown complete.
05:09:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskqezt4V/test-tmp/mini-kudu-cluster452044082605959630/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:09:31.732 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.60:46141(127.1.18.60:46141)] connecting to peer
05:09:31.742 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:31.784 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.18.60:46141(127.1.18.60:46141)] Successfully connected to peer
05:09:31.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:31.795 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:31.797 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:31.800 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:31.800 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:31.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:31.906 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
05:09:31.916 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.18.60
05:09:32.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:32 dist-test-slave-5kl8 krb5kdc[1108](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658565, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.60@KRBTEST.COM
05:09:32.200 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf2e5c16b, L:/127.0.0.1:43504 - R:/127.1.18.60:46141] using SASL/GSSAPI
05:09:32.222 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:32.245 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:32.252 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:32.256 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.256 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.257 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.260 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.261 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:32.263 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:32.265 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:32.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.273 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.275 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.276 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.276 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:32.283 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:32.284 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:32.288 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.288 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.292 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.292 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.294 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:32.303 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:32.305 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:32.308 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.308 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.311 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:32.343 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:32.345 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:32.348 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.348 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.348 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.348 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.352 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:32.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:32.354207 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:32.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:32.354477 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:32.423 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:32.425 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:32.428 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.428 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.433 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.435 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.436 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:32.563 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:32.566 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:32.568 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.568 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.571 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.572 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.573 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:32.703 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:32.705 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:32.707 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.707 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.711 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.712 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.712 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.713 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:32.863 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:32.865 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:32.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:32.871 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:32.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:33.357817 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:33.358117 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:33.703 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:33.705 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:33.708 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:33.708 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:33.708 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:33.708 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:33.708 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:33.708 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:33.710 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:33.763 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:33.766 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:33.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:33.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:33.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:33.770 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:33.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:33.771 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:33.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:34.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:34.361445 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:34.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:34.361723 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:34.377233 1454 master_runner.cc:186] Error getting master registration for 127.1.18.62:43233: Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111),
05:09:34.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:34.377800 1454 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:09:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:34.481612 1628 raft_consensus.cc:482] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:09:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:34.482121 1628 raft_consensus.cc:504] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } }
05:09:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:34.484391 1628 leader_election.cc:288] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233), a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831)
05:09:34.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:34.493492 1477 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831): Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:34.498097 1476 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233): Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111)
05:09:34.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:34.498518 1476 leader_election.cc:302] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bdd24dfef5e941b89b299c1fa6cb1b82; no voters: 2cebbb18a87c4a45ab0f0bedff200069, a07e9956d3454ce2aaf019323f695f8f
05:09:34.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:34.499186 1628 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:09:35.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:35.364809 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:35.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:35.365092 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:36.368531 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:36.369205 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:37.068790 1635 raft_consensus.cc:482] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:09:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:37.069249 1635 raft_consensus.cc:504] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } }
05:09:37.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:37.071375 1635 leader_election.cc:288] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233), a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831)
05:09:37.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:37.073361 1476 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111) [suppressed 5 similar messages]
05:09:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:37.076805 1476 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233): Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111)
05:09:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:37.077294 1477 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831): Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:37.077643 1477 leader_election.cc:302] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bdd24dfef5e941b89b299c1fa6cb1b82; no voters: 2cebbb18a87c4a45ab0f0bedff200069, a07e9956d3454ce2aaf019323f695f8f
05:09:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:37.078331 1635 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:09:37.283 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:37.284 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:37.286 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:37.287 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:37.287 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:37.287 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:37.287 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:37.287 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:37.288 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:37.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:37.372968 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:37.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:37.373461 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:37.463 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:37.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:37.467 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:37.467 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:37.468 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:37.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:37.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:37.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:37.470 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:38.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:38.377218 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:38.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:38.377562 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:38.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:38.827123 1643 raft_consensus.cc:482] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:09:38.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:38.827636 1643 raft_consensus.cc:504] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } }
05:09:38.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:38.829566 1643 leader_election.cc:288] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233), a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831)
05:09:38.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:38.835227 1477 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831): Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:38.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:38.835834 1476 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233): Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111)
05:09:38.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:38.836267 1476 leader_election.cc:302] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bdd24dfef5e941b89b299c1fa6cb1b82; no voters: 2cebbb18a87c4a45ab0f0bedff200069, a07e9956d3454ce2aaf019323f695f8f
05:09:38.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:38.836977 1643 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:09:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:39 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658579, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:39.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:39.046725 1469 init.cc:295] Successfully reacquired a new kerberos TGT
05:09:39.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:39.381145 1454 master_runner.cc:186] Error getting master registration for 127.1.18.62:43233: Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111),
05:09:39.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:39.381356 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:39.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:39.381704 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:39.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:39.381737 1454 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:09:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:40.385165 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:40.385499 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:41.303 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:41.304 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:41.306 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:41.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:41.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:41.310 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:41.310 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:41.310 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:41.311 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:41.363 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:41.365 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:41.367 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:41.367 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:41.367 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:41.367 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:41.367 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:41.367 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:41.369 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:41.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:41.388170 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:41.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:41.388501 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:41.883 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:41.885 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:41.887 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:41.887 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:41.888 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:41.888 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:41.888 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:41.888 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:41.890 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:42.022 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:42.024 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:42.027 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:42.028 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:42.028 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:42.028 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:42.028 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:42.028 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:42.029 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:42.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:42.391330 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:42.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:42.391601 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:43.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:43.394456 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:43.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:43.394729 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:44.183 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:44.185 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:44.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:44.187 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:44.187 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:44.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:44.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:44.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:44.191 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:44.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:44.384510 1454 master_runner.cc:186] Error getting master registration for 127.1.18.62:43233: Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111),
05:09:44.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:44.384891 1454 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:09:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:44.397585 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:44.397878 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:45.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:45.400827 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:45.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:45.401114 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:45.423 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:45.425 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:45.427 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:45.428 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:45.428 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:45.428 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:45.428 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:45.428 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:45.430 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:45.542 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:45.544 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:45.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:45.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:45.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:45.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:45.547 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:45.547 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:45.548 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:46.403945 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:46.404222 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:47.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:47.407328 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:47.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:47.407609 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:48.322 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:48.323 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:48.324 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:48.324 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:48.324 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:48.324 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:48.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:48.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:48.328 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:48.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:48.411043 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:48.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:48.411586 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:49.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:49 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658589, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:49.049943 1469 init.cc:295] Successfully reacquired a new kerberos TGT
05:09:49.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:49.388448 1454 master_runner.cc:186] Error getting master registration for 127.1.18.62:43233: Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111),
05:09:49.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:49.389025 1454 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:09:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:49.415019 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:49.415316 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:50.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:50.114621 1661 raft_consensus.cc:482] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:09:50.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:50.115180 1661 raft_consensus.cc:504] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2cebbb18a87c4a45ab0f0bedff200069" member_type: VOTER last_known_addr { host: "127.1.18.62" port: 43233 } } peers { permanent_uuid: "a07e9956d3454ce2aaf019323f695f8f" member_type: VOTER last_known_addr { host: "127.1.18.61" port: 46831 } } peers { permanent_uuid: "bdd24dfef5e941b89b299c1fa6cb1b82" member_type: VOTER last_known_addr { host: "127.1.18.60" port: 46141 } }
05:09:50.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:50.117015 1661 leader_election.cc:288] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233), a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831)
05:09:50.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:50.119128 1476 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111) [suppressed 3 similar messages]
05:09:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:50.122535 1477 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a07e9956d3454ce2aaf019323f695f8f (127.1.18.61:46831): Network error: Client connection negotiation failed: client connection to 127.1.18.61:46831: connect: Connection refused (error 111)
05:09:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:50.123035 1476 leader_election.cc:334] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2cebbb18a87c4a45ab0f0bedff200069 (127.1.18.62:43233): Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111)
05:09:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:50.123409 1476 leader_election.cc:302] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bdd24dfef5e941b89b299c1fa6cb1b82; no voters: 2cebbb18a87c4a45ab0f0bedff200069, a07e9956d3454ce2aaf019323f695f8f
05:09:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:50.124234 1661 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:09:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:50.418409 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:50.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:50.418685 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:51.421729 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:51.422019 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:51.603 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:51.604 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:51.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:51.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:51.606 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:51.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:51.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:51.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:51.608 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:52.425064 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:52.425361 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:52.623 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:52.624 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:52.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:52.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:52.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:52.626 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:52.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:52.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:52.628 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:53.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:53.428475 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:53.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:53.428767 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:54.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:54.391891 1454 master_runner.cc:186] Error getting master registration for 127.1.18.62:43233: Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111),
05:09:54.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:54.392242 1454 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:09:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:54.431674 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:54.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:54.431977 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:55.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:55.434769 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:55.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:55.435042 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:55.602 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:55.603 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:55.604 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:55.604 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:55.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:55.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:55.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:55.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:55.607 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:55.903 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:55.904 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:55.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:55.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:55.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:55.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:55.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:55.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:55.908 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:09:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:56.438169 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:56.438463 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:57.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:57.441915 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:57.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:57.442203 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:58.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:58.446063 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:58.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:58.446412 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 05:09:59 dist-test-slave-5kl8 krb5kdc[1108](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790658599, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:09:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:59.052353 1469 init.cc:295] Successfully reacquired a new kerberos TGT
05:09:59.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:59.395910 1454 master_runner.cc:186] Error getting master registration for 127.1.18.62:43233: Network error: Client connection negotiation failed: client connection to 127.1.18.62:43233: connect: Connection refused (error 111),
05:09:59.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:09:59.396303 1454 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:09:59.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:59.449108 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:09:59.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:09:59.449409 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:09:59.483 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:09:59.484 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:09:59.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:59.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:59.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:09:59.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:59.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:59.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:09:59.488 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:10:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:10:00.451936 1540 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdd24dfef5e941b89b299c1fa6cb1b82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:10:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 05:10:00.452217 1540 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:10:01.403 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.62:43233(127.1.18.62:43233)] connecting to peer
05:10:01.404 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.61:46831(127.1.18.61:46831)] connecting to peer
05:10:01.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:10:01.405 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:10:01.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.61:46831(127.1.18.61:46831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:10:01.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.62:43233(127.1.18.62:43233)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:10:01.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.61:46831: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831
05:10:01.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.62:43233: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
05:10:01.408 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831; will retry
05:10:01.413 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29927), Traces: [0ms] refreshing cache from master, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [230ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [285ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [296ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [707ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [712ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [732ms] refreshing cache from master, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [746ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [747ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [750ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [752ms] refreshing cache from master, [752ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [753ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [771ms] refreshing cache from master, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [777ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [783ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [792ms] refreshing cache from master, [792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [793ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [797ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [797ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [831ms] refreshing cache from master, [832ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [837ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [837ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [841ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [912ms] refreshing cache from master, [912ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [917ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [923ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [925ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1051ms] refreshing cache from master, [1052ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1057ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1062ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1191ms] refreshing cache from master, [1192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1203ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1351ms] refreshing cache from master, [1352ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1354ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1358ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1361ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1363ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2192ms] refreshing cache from master, [2192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [2193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [2194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [2197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [2197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2199ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [2199ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2251ms] refreshing cache from master, [2251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [2252ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [2254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [2258ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [2260ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2262ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [2263ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [5771ms] refreshing cache from master, [5771ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [5772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [5773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [5776ms] too many traces: truncated at 100 traces, deferred=Deferred@329602699(state=PENDING, result=null, callback=(continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417) -> (continuation of Deferred@209584610 after retry RPC after error@1578152780) -> (continuation of Deferred@1340221410 after retry RPC after error@295718810) -> (continuation of Deferred@1831681373 after retry RPC after error@914123111) -> (continuation of Deferred@1761580413 after retry RPC after error@2075183975) -> (continuation of Deferred@2138298009 after retry RPC after error@705806145) -> (continuation of Deferred@1045533443 after retry RPC after error@1128762461) -> (continuation of Deferred@576802226 after retry RPC after error@86840584) -> (continuation of Deferred@928401189 after retry RPC after error@1217134838) -> (continuation of Deferred@1621674120 after retry RPC after error@230687936) -> (continuation of Deferred@136971570 after retry RPC after error@537472881) -> (continuation of Deferred@105074544 after retry RPC after error@195607932) -> (continuation of Deferred@1400060590 after retry RPC after error@979952918) -> (continuation of Deferred@650909156 after retry RPC after error@1749098268) -> (continuation of Deferred@573439728 after retry RPC after error@1594949622) -> (continuation of Deferred@431338181 after retry RPC after error@2037184272) -> (continuation of Deferred@880499891 after retry RPC after error@1825368709), errback=(continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417) -> (continuation of Deferred@209584610 after retry RPC after error@1578152780) -> (continuation of Deferred@1340221410 after retry RPC after error@295718810) -> (continuation of Deferred@1831681373 after retry RPC after error@914123111) -> (continuation of Deferred@1761580413 after retry RPC after error@2075183975) -> (continuation of Deferred@2138298009 after retry RPC after error@705806145) -> (continuation of Deferred@1045533443 after retry RPC after error@1128762461) -> (continuation of Deferred@576802226 after retry RPC after error@86840584) -> (continuation of Deferred@928401189 after retry RPC after error@1217134838) -> (continuation of Deferred@1621674120 after retry RPC after error@230687936) -> (continuation of Deferred@136971570 after retry RPC after error@537472881) -> (continuation of Deferred@105074544 after retry RPC after error@195607932) -> (continuation of Deferred@1400060590 after retry RPC after error@979952918) -> (continuation of Deferred@650909156 after retry RPC after error@1749098268) -> (continuation of Deferred@573439728 after retry RPC after error@1594949622) -> (continuation of Deferred@431338181 after retry RPC after error@2037184272) -> (continuation of Deferred@880499891 after retry RPC after error@1825368709)))
05:10:01.418 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.60:46141(127.1.18.60:46141)] disconnecting while in state READY
05:10:01.420 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.61:46831(127.1.18.61:46831)] disconnecting while in state TERMINATED
05:10:01.420 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.62:43233(127.1.18.62:43233)] disconnecting while in state TERMINATED
05:10:01.425 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:10:01.427 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.18.60:46141(127.1.18.60:46141)] handling channelInactive
05:10:01.427 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.60:46141(127.1.18.60:46141)] cleaning up while in state READY due to: connection closed
05:10:01.428 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:10:01.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:10:01.435201 1096 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu with pid 1454
05:10:02.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:10:02Z chronyd exiting
05:10:03.578 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:10:03.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:10:03.579385 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu with pid 673
05:10:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:10:03.601543 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu with pid 812
05:10:03.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:10:03.622660 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu with pid 953
05:10:03.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:10:03.646041 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu with pid 386
05:10:03.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:10:03.674041 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu with pid 456
05:10:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 05:10:03.700198 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqezt4V/build/tsan/bin/kudu with pid 531
05:10:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T05:10:04Z chronyd exiting
05:10:05.808 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29927), Traces: [0ms] refreshing cache from master, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [230ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [285ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [296ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [707ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [712ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [732ms] refreshing cache from master, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [746ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [747ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [750ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [752ms] refreshing cache from master, [752ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [753ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [771ms] refreshing cache from master, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [777ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [783ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [792ms] refreshing cache from master, [792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [793ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [797ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [797ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [831ms] refreshing cache from master, [832ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [837ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [837ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [841ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [912ms] refreshing cache from master, [912ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [917ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [923ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [925ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1051ms] refreshing cache from master, [1052ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1057ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1062ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1191ms] refreshing cache from master, [1192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1203ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1351ms] refreshing cache from master, [1352ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1354ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1358ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1361ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1363ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2192ms] refreshing cache from master, [2192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [2193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [2194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [2197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [2197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2199ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [2199ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2251ms] refreshing cache from master, [2251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [2252ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [2254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [2258ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [2260ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2262ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [2263ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [5771ms] refreshing cache from master, [5771ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [5772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [5773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [5776ms] too many traces: truncated at 100 traces, deferred=Deferred@329602699(state=PENDING, result=null, callback=(continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417) -> (continuation of Deferred@209584610 after retry RPC after error@1578152780) -> (continuation of Deferred@1340221410 after retry RPC after error@295718810) -> (continuation of Deferred@1831681373 after retry RPC after error@914123111) -> (continuation of Deferred@1761580413 after retry RPC after error@2075183975) -> (continuation of Deferred@2138298009 after retry RPC after error@705806145) -> (continuation of Deferred@1045533443 after retry RPC after error@1128762461) -> (continuation of Deferred@576802226 after retry RPC after error@86840584) -> (continuation of Deferred@928401189 after retry RPC after error@1217134838) -> (continuation of Deferred@1621674120 after retry RPC after error@230687936) -> (continuation of Deferred@136971570 after retry RPC after error@537472881) -> (continuation of Deferred@105074544 after retry RPC after error@195607932) -> (continuation of Deferred@1400060590 after retry RPC after error@979952918) -> (continuation of Deferred@650909156 after retry RPC after error@1749098268) -> (continuation of Deferred@573439728 after retry RPC after error@1594949622) -> (continuation of Deferred@431338181 after retry RPC after error@2037184272) -> (continuation of Deferred@880499891 after retry RPC after error@1825368709), errback=(continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417) -> (continuation of Deferred@209584610 after retry RPC after error@1578152780) -> (continuation of Deferred@1340221410 after retry RPC after error@295718810) -> (continuation of Deferred@1831681373 after retry RPC after error@914123111) -> (continuation of Deferred@1761580413 after retry RPC after error@2075183975) -> (continuation of Deferred@2138298009 after retry RPC after error@705806145) -> (continuation of Deferred@1045533443 after retry RPC after error@1128762461) -> (continuation of Deferred@576802226 after retry RPC after error@86840584) -> (continuation of Deferred@928401189 after retry RPC after error@1217134838) -> (continuation of Deferred@1621674120 after retry RPC after error@230687936) -> (continuation of Deferred@136971570 after retry RPC after error@537472881) -> (continuation of Deferred@105074544 after retry RPC after error@195607932) -> (continuation of Deferred@1400060590 after retry RPC after error@979952918) -> (continuation of Deferred@650909156 after retry RPC after error@1749098268) -> (continuation of Deferred@573439728 after retry RPC after error@1594949622) -> (continuation of Deferred@431338181 after retry RPC after error@2037184272) -> (continuation of Deferred@880499891 after retry RPC after error@1825368709)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.62:43233
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29927), Traces: [0ms] refreshing cache from master, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [230ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [285ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [296ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [707ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [712ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [732ms] refreshing cache from master, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [746ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [747ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [750ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [752ms] refreshing cache from master, [752ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [753ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [771ms] refreshing cache from master, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [777ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [783ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [792ms] refreshing cache from master, [792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [793ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [797ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [797ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [831ms] refreshing cache from master, [832ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [837ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [837ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [841ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [912ms] refreshing cache from master, [912ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [917ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [923ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [925ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1051ms] refreshing cache from master, [1052ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1057ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1062ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1191ms] refreshing cache from master, [1192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1203ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1351ms] refreshing cache from master, [1352ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1354ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1358ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1361ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1363ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2192ms] refreshing cache from master, [2192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [2193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [2194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [2197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [2197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2199ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [2199ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2251ms] refreshing cache from master, [2251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [2252ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [2254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [2258ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [2260ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2262ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [2263ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [5771ms] refreshing cache from master, [5771ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [5772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [5773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [5776ms] too many traces: truncated at 100 traces, deferred=Deferred@329602699(state=PENDING, result=null, callback=(continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417) -> (continuation of Deferred@209584610 after retry RPC after error@1578152780) -> (continuation of Deferred@1340221410 after retry RPC after error@295718810) -> (continuation of Deferred@1831681373 after retry RPC after error@914123111) -> (continuation of Deferred@1761580413 after retry RPC after error@2075183975) -> (continuation of Deferred@2138298009 after retry RPC after error@705806145) -> (continuation of Deferred@1045533443 after retry RPC after error@1128762461) -> (continuation of Deferred@576802226 after retry RPC after error@86840584) -> (continuation of Deferred@928401189 after retry RPC after error@1217134838) -> (continuation of Deferred@1621674120 after retry RPC after error@230687936) -> (continuation of Deferred@136971570 after retry RPC after error@537472881) -> (continuation of Deferred@105074544 after retry RPC after error@195607932) -> (continuation of Deferred@1400060590 after retry RPC after error@979952918) -> (continuation of Deferred@650909156 after retry RPC after error@1749098268) -> (continuation of Deferred@573439728 after retry RPC after error@1594949622) -> (continuation of Deferred@431338181 after retry RPC after error@2037184272) -> (continuation of Deferred@880499891 after retry RPC after error@1825368709), errback=(continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417) -> (continuation of Deferred@209584610 after retry RPC after error@1578152780) -> (continuation of Deferred@1340221410 after retry RPC after error@295718810) -> (continuation of Deferred@1831681373 after retry RPC after error@914123111) -> (continuation of Deferred@1761580413 after retry RPC after error@2075183975) -> (continuation of Deferred@2138298009 after retry RPC after error@705806145) -> (continuation of Deferred@1045533443 after retry RPC after error@1128762461) -> (continuation of Deferred@576802226 after retry RPC after error@86840584) -> (continuation of Deferred@928401189 after retry RPC after error@1217134838) -> (continuation of Deferred@1621674120 after retry RPC after error@230687936) -> (continuation of Deferred@136971570 after retry RPC after error@537472881) -> (continuation of Deferred@105074544 after retry RPC after error@195607932) -> (continuation of Deferred@1400060590 after retry RPC after error@979952918) -> (continuation of Deferred@650909156 after retry RPC after error@1749098268) -> (continuation of Deferred@573439728 after retry RPC after error@1594949622) -> (continuation of Deferred@431338181 after retry RPC after error@2037184272) -> (continuation of Deferred@880499891 after retry RPC after error@1825368709)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.62:43233
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
05:10:06.369 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
05:10:06.370 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29927), Traces: [0ms] refreshing cache from master, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [230ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [285ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [296ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [707ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [712ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [732ms] refreshing cache from master, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [746ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [747ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [750ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [752ms] refreshing cache from master, [752ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [753ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [771ms] refreshing cache from master, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [777ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [783ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [792ms] refreshing cache from master, [792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [793ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [797ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [797ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [831ms] refreshing cache from master, [832ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [837ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [837ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [841ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [912ms] refreshing cache from master, [912ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [917ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [923ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [925ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1051ms] refreshing cache from master, [1052ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1057ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1062ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1191ms] refreshing cache from master, [1192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1203ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1351ms] refreshing cache from master, [1352ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1354ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1358ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1361ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1363ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2192ms] refreshing cache from master, [2192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [2193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [2194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [2197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [2197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2199ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [2199ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2251ms] refreshing cache from master, [2251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [2252ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [2254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [2258ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [2260ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2262ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [2263ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [5771ms] refreshing cache from master, [5771ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [5772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [5773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [5776ms] too many traces: truncated at 100 traces, deferred=Deferred@329602699(state=PENDING, result=null, callback=(continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417) -> (continuation of Deferred@209584610 after retry RPC after error@1578152780) -> (continuation of Deferred@1340221410 after retry RPC after error@295718810) -> (continuation of Deferred@1831681373 after retry RPC after error@914123111) -> (continuation of Deferred@1761580413 after retry RPC after error@2075183975) -> (continuation of Deferred@2138298009 after retry RPC after error@705806145) -> (continuation of Deferred@1045533443 after retry RPC after error@1128762461) -> (continuation of Deferred@576802226 after retry RPC after error@86840584) -> (continuation of Deferred@928401189 after retry RPC after error@1217134838) -> (continuation of Deferred@1621674120 after retry RPC after error@230687936) -> (continuation of Deferred@136971570 after retry RPC after error@537472881) -> (continuation of Deferred@105074544 after retry RPC after error@195607932) -> (continuation of Deferred@1400060590 after retry RPC after error@979952918) -> (continuation of Deferred@650909156 after retry RPC after error@1749098268) -> (continuation of Deferred@573439728 after retry RPC after error@1594949622) -> (continuation of Deferred@431338181 after retry RPC after error@2037184272) -> (continuation of Deferred@880499891 after retry RPC after error@1825368709), errback=(continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417) -> (continuation of Deferred@209584610 after retry RPC after error@1578152780) -> (continuation of Deferred@1340221410 after retry RPC after error@295718810) -> (continuation of Deferred@1831681373 after retry RPC after error@914123111) -> (continuation of Deferred@1761580413 after retry RPC after error@2075183975) -> (continuation of Deferred@2138298009 after retry RPC after error@705806145) -> (continuation of Deferred@1045533443 after retry RPC after error@1128762461) -> (continuation of Deferred@576802226 after retry RPC after error@86840584) -> (continuation of Deferred@928401189 after retry RPC after error@1217134838) -> (continuation of Deferred@1621674120 after retry RPC after error@230687936) -> (continuation of Deferred@136971570 after retry RPC after error@537472881) -> (continuation of Deferred@105074544 after retry RPC after error@195607932) -> (continuation of Deferred@1400060590 after retry RPC after error@979952918) -> (continuation of Deferred@650909156 after retry RPC after error@1749098268) -> (continuation of Deferred@573439728 after retry RPC after error@1594949622) -> (continuation of Deferred@431338181 after retry RPC after error@2037184272) -> (continuation of Deferred@880499891 after retry RPC after error@1825368709)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.62:43233
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=29927), Traces: [0ms] refreshing cache from master, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [230ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [285ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [296ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [707ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [712ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [732ms] refreshing cache from master, [733ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [746ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [747ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [750ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [752ms] refreshing cache from master, [752ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [753ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [754ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [771ms] refreshing cache from master, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [777ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [783ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [792ms] refreshing cache from master, [792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [793ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [797ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [797ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [800ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [831ms] refreshing cache from master, [832ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [833ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [837ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [837ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [841ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [912ms] refreshing cache from master, [912ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [917ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [923ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [925ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1051ms] refreshing cache from master, [1052ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1054ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1057ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1062ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1191ms] refreshing cache from master, [1192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1203ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1351ms] refreshing cache from master, [1352ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [1353ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [1354ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [1358ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [1361ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [1363ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2192ms] refreshing cache from master, [2192ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [2193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [2194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [2197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [2197ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2199ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [2199ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2251ms] refreshing cache from master, [2251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [2252ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [2254ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [2258ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.61:46831: Network error: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831, [2260ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.62:43233: Network error: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [2262ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.60:46141: OK, [2263ms] delaying RPC due to: Service unavailable: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233, [5771ms] refreshing cache from master, [5771ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.62:43233, [5772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.60:46141, [5773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.61:46831, [5776ms] too many traces: truncated at 100 traces, deferred=Deferred@329602699(state=PENDING, result=null, callback=(continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417) -> (continuation of Deferred@209584610 after retry RPC after error@1578152780) -> (continuation of Deferred@1340221410 after retry RPC after error@295718810) -> (continuation of Deferred@1831681373 after retry RPC after error@914123111) -> (continuation of Deferred@1761580413 after retry RPC after error@2075183975) -> (continuation of Deferred@2138298009 after retry RPC after error@705806145) -> (continuation of Deferred@1045533443 after retry RPC after error@1128762461) -> (continuation of Deferred@576802226 after retry RPC after error@86840584) -> (continuation of Deferred@928401189 after retry RPC after error@1217134838) -> (continuation of Deferred@1621674120 after retry RPC after error@230687936) -> (continuation of Deferred@136971570 after retry RPC after error@537472881) -> (continuation of Deferred@105074544 after retry RPC after error@195607932) -> (continuation of Deferred@1400060590 after retry RPC after error@979952918) -> (continuation of Deferred@650909156 after retry RPC after error@1749098268) -> (continuation of Deferred@573439728 after retry RPC after error@1594949622) -> (continuation of Deferred@431338181 after retry RPC after error@2037184272) -> (continuation of Deferred@880499891 after retry RPC after error@1825368709), errback=(continuation of Deferred@1038598860 after retry RPC after error@613580517) -> (continuation of Deferred@2011467001 after retry RPC after error@1313639808) -> (continuation of Deferred@1295963943 after retry RPC after error@2053963059) -> (continuation of Deferred@691807683 after retry RPC after error@53906921) -> (continuation of Deferred@948626759 after retry RPC after error@630325760) -> (continuation of Deferred@1833449804 after retry RPC after error@1520871625) -> (continuation of Deferred@896126471 after retry RPC after error@14571688) -> (continuation of Deferred@745575334 after retry RPC after error@1132430849) -> (continuation of Deferred@949838239 after retry RPC after error@1257534780) -> (continuation of Deferred@1647387239 after retry RPC after error@1572994990) -> (continuation of Deferred@235142160 after retry RPC after error@1731671417) -> (continuation of Deferred@209584610 after retry RPC after error@1578152780) -> (continuation of Deferred@1340221410 after retry RPC after error@295718810) -> (continuation of Deferred@1831681373 after retry RPC after error@914123111) -> (continuation of Deferred@1761580413 after retry RPC after error@2075183975) -> (continuation of Deferred@2138298009 after retry RPC after error@705806145) -> (continuation of Deferred@1045533443 after retry RPC after error@1128762461) -> (continuation of Deferred@576802226 after retry RPC after error@86840584) -> (continuation of Deferred@928401189 after retry RPC after error@1217134838) -> (continuation of Deferred@1621674120 after retry RPC after error@230687936) -> (continuation of Deferred@136971570 after retry RPC after error@537472881) -> (continuation of Deferred@105074544 after retry RPC after error@195607932) -> (continuation of Deferred@1400060590 after retry RPC after error@979952918) -> (continuation of Deferred@650909156 after retry RPC after error@1749098268) -> (continuation of Deferred@573439728 after retry RPC after error@1594949622) -> (continuation of Deferred@431338181 after retry RPC after error@2037184272) -> (continuation of Deferred@880499891 after retry RPC after error@1825368709)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.62:43233,127.1.18.60:46141,127.1.18.61:46831) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.61:46831(127.1.18.61:46831): Connection refused: /127.1.18.61:46831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.62:43233(127.1.18.62:43233): Connection refused: /127.1.18.62:43233
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.62:43233
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)